evalscope 0.17.1__py3-none-any.whl → 1.0.1__py3-none-any.whl

evalscope/app/ui/multi_model.py

@@ -10,8 +10,14 @@ from typing import TYPE_CHECKING
 from evalscope.report import ReportKey, get_data_frame
 from evalscope.utils.logger import get_logger
 from ..constants import LATEX_DELIMITERS, MODEL_TOKEN, REPORT_TOKEN
-from ..utils.data_utils import (get_acc_report_df, get_compare_report_df, get_model_prediction, get_single_dataset_df,
-                                load_multi_report, load_single_report)
+from ..utils.data_utils import (
+    get_acc_report_df,
+    get_compare_report_df,
+    get_model_prediction,
+    get_single_dataset_df,
+    load_multi_report,
+    load_single_report,
+)
 from ..utils.localization import get_multi_model_locale
 from ..utils.text_utils import convert_markdown_image, process_model_prediction
 from ..utils.visualization import plot_multi_report_radar
@@ -62,7 +68,8 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
                 label=locale_dict.get('answer_mode'),
                 choices=['All', 'Pass A & B', 'Fail A & B', 'Pass A, Fail B', 'Fail A, Pass B'],
                 value='All',
-                interactive=True)
+                interactive=True
+            )
             score_threshold = gr.Number(value=0.99, label=locale_dict['score_threshold'], interactive=True)
 
         data_comparison_df = gr.State(None)
@@ -75,7 +82,8 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
                 comparison_counts = gr.Markdown('')
             with gr.Column():
                 page_number = gr.Number(
-                    value=1, label=locale_dict['page'], minimum=1, maximum=1, step=1, interactive=True)
+                    value=1, label=locale_dict['page'], minimum=1, maximum=1, step=1, interactive=True
+                )
 
         # Input and Gold answer sections remain at the top
         with gr.Row(variant='panel'):
@@ -133,7 +141,8 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
 
     @multi_report_name.change(
         inputs=[sidebar.root_path, multi_report_name],
-        outputs=[report_list, radar_plot, score_table, model_a_select, model_b_select])
+        outputs=[report_list, radar_plot, score_table, model_a_select, model_b_select]
+    )
     def update_multi_report_data(root_path, multi_report_names):
         if not multi_report_names:
             return gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip()
@@ -147,13 +156,14 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
         model_choices = multi_report_names
 
         return report_list, report_radar_plot, styler, gr.update(
-            choices=model_choices, value=model_choices[0]), gr.update(
-                choices=model_choices, value=model_choices[1] if len(model_choices) > 1 else None)
+            choices=model_choices, value=model_choices[0]
+        ), gr.update(choices=model_choices, value=model_choices[1] if len(model_choices) > 1 else None)
 
     @gr.on(
         triggers=[model_a_select.change, model_b_select.change],
         inputs=[sidebar.root_path, model_a_select, model_b_select],
-        outputs=[model_a_report, model_b_report, model_a_dir, model_b_dir, model_a_name, model_b_name, dataset_radio])
+        outputs=[model_a_report, model_b_report, model_a_dir, model_b_dir, model_a_name, model_b_name, dataset_radio]
+    )
     def update_selected_models(root_path, model_a, model_b):
         if not model_a or not model_b:
             return gr.skip()
@@ -172,13 +182,16 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
         model_a_name = model_a.split(REPORT_TOKEN)[1].split(MODEL_TOKEN)[0]
         model_b_name = model_b.split(REPORT_TOKEN)[1].split(MODEL_TOKEN)[0]
 
-        return (model_a_reports, model_b_reports, model_a_dir, model_b_dir, model_a_name, model_b_name,
-                gr.update(choices=common_datasets, value=common_datasets[0] if common_datasets else None))
+        return (
+            model_a_reports, model_b_reports, model_a_dir, model_b_dir, model_a_name, model_b_name,
+            gr.update(choices=common_datasets, value=common_datasets[0] if common_datasets else None)
+        )
 
     @gr.on(
         triggers=[dataset_radio.change],
         inputs=[dataset_radio, model_a_report, model_b_report],
-        outputs=[subset_select, data_comparison_df])
+        outputs=[subset_select, data_comparison_df]
+    )
     def update_dataset_comparison(dataset_name, model_a_report, model_b_report):
         if not dataset_name or model_a_report is None or model_b_report is None:
             return gr.skip()
@@ -198,7 +211,8 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
     @gr.on(
         triggers=[subset_select.change],
         inputs=[model_a_dir, model_b_dir, model_a_name, model_b_name, dataset_radio, subset_select],
-        outputs=[data_comparison_df, page_number])
+        outputs=[data_comparison_df, page_number]
+    )
     def update_comparison_data(model_a_dir, model_b_dir, model_a_name, model_b_name, dataset_name, subset_name):
         if not subset_name or not dataset_name:
             return gr.skip()
@@ -230,7 +244,8 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
     @gr.on(
         triggers=[data_comparison_df.change, answer_mode_radio.change, score_threshold.change],
         inputs=[data_comparison_df, answer_mode_radio, score_threshold],
-        outputs=[filtered_comparison_df, page_number, comparison_counts])
+        outputs=[filtered_comparison_df, page_number, comparison_counts]
+    )
     def filter_comparison_data(comparison_df, answer_mode, score_threshold):
         if comparison_df is None:
             return None, gr.update(value=1, maximum=1), ''
@@ -256,13 +271,19 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
         # Count statistics
         pass_a_count = len(comparison_df[comparison_df['A_NScore'] >= score_threshold])
         pass_b_count = len(comparison_df[comparison_df['B_NScore'] >= score_threshold])
-        pass_both_count = len(comparison_df[(comparison_df['A_NScore'] >= score_threshold)
-                                            & (comparison_df['B_NScore'] >= score_threshold)])
-        fail_both_count = len(comparison_df[(comparison_df['A_NScore'] < score_threshold)
-                                            & (comparison_df['B_NScore'] < score_threshold)])
-
-        counts_text = (f'### All: {all_count} | Pass A: {pass_a_count} | Pass B: {pass_b_count} | '
-                       f'Pass Both: {pass_both_count} | Fail Both: {fail_both_count}')
+        pass_both_count = len(
+            comparison_df[(comparison_df['A_NScore'] >= score_threshold)
+                          & (comparison_df['B_NScore'] >= score_threshold)]
+        )
+        fail_both_count = len(
+            comparison_df[(comparison_df['A_NScore'] < score_threshold)
+                          & (comparison_df['B_NScore'] < score_threshold)]
+        )
+
+        counts_text = (
+            f'### All: {all_count} | Pass A: {pass_a_count} | Pass B: {pass_b_count} | '
+            f'Pass Both: {pass_both_count} | Fail Both: {fail_both_count}'
+        )
 
         max_page = max(1, len(filtered_df))
 
@@ -277,9 +298,11 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
         outputs=[
             input_text, gold_text, model_a_generated, model_a_pred, model_a_score, model_a_nscore, model_b_generated,
             model_b_pred, model_b_score, model_b_nscore
-        ])
-    def update_comparison_display(filtered_df, page_number, score_threshold, model_a_select, model_b_select,
-                                  model_a_name_val, model_b_name_val):
+        ]
+    )
+    def update_comparison_display(
+        filtered_df, page_number, score_threshold, model_a_select, model_b_select, model_a_name_val, model_b_name_val
+    ):
         if filtered_df is None or len(filtered_df) == 0:
             return '', '', '', '', '', '', '', '', '', ''
 
@@ -317,7 +340,9 @@ def create_multi_model_tab(sidebar: 'SidebarComponents', lang: str):
         else:
             b_nscore_html = f"<div style='background-color:rgb(151, 31, 44); padding:10px;'>{b_nscore_val}</div>"
 
-        return (input_md, gold_md, a_generated_md, a_pred_md, a_score_md, a_nscore_html, b_generated_md, b_pred_md,
-                b_score_md, b_nscore_html)
+        return (
+            input_md, gold_md, a_generated_md, a_pred_md, a_score_md, a_nscore_html, b_generated_md, b_pred_md,
+            b_score_md, b_nscore_html
+        )
 
     return MultiModelComponents(multi_report_name=multi_report_name)

evalscope/app/ui/single_model.py

@@ -10,8 +10,13 @@ from typing import TYPE_CHECKING
 from evalscope.report import Report, ReportKey, get_data_frame
 from evalscope.utils.logger import get_logger
 from ..constants import DATASET_TOKEN, LATEX_DELIMITERS, MODEL_TOKEN, REPORT_TOKEN
-from ..utils.data_utils import (get_acc_report_df, get_model_prediction, get_report_analysis, get_single_dataset_df,
-                                load_single_report)
+from ..utils.data_utils import (
+    get_acc_report_df,
+    get_model_prediction,
+    get_report_analysis,
+    get_single_dataset_df,
+    load_single_report,
+)
 from ..utils.localization import get_single_model_locale
 from ..utils.text_utils import convert_markdown_image, process_json_content, process_model_prediction
 from ..utils.visualization import plot_single_dataset_scores, plot_single_report_scores, plot_single_report_sunburst
@@ -63,7 +68,8 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
 
         with gr.Row():
             answer_mode_radio = gr.Radio(
-                label=locale_dict['answer_mode'], choices=['All', 'Pass', 'Fail'], value='All', interactive=True)
+                label=locale_dict['answer_mode'], choices=['All', 'Pass', 'Fail'], value='All', interactive=True
+            )
             score_threshold = gr.Number(value=0.99, label=locale_dict['score_threshold'], interactive=True)
 
         data_review_df = gr.State(None)
@@ -76,7 +82,8 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
                 answer_mode_counts = gr.Markdown('')
             with gr.Column():
                 page_number = gr.Number(
-                    value=1, label=locale_dict['page'], minimum=1, maximum=1, step=1, interactive=True)
+                    value=1, label=locale_dict['page'], minimum=1, maximum=1, step=1, interactive=True
+                )
 
         # show data review table
         with gr.Row(variant='panel'):
@@ -98,14 +105,15 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
         with gr.Row(variant='panel'):
             with gr.Column():
                 gr.Markdown('### *Input*')
-                input_text = gr.Code('', elem_id='input_text', language='json', wrap_lines=False)
+                input_text = gr.Markdown('', elem_id='input_text', latex_delimiters=LATEX_DELIMITERS)
             with gr.Column():
                 gr.Markdown('### *Generated*')
                 generated_text = gr.Markdown('', elem_id='generated_text', latex_delimiters=LATEX_DELIMITERS)
 
     @report_name.change(
         inputs=[sidebar.root_path, report_name],
-        outputs=[report_list, task_config, dataset_radio, work_dir, model_name])
+        outputs=[report_list, task_config, dataset_radio, work_dir, model_name]
+    )
     def update_single_report_data(root_path, report_name):
         report_list, datasets, task_cfg = load_single_report(root_path, report_name)
         work_dir = os.path.join(root_path, report_name.split(REPORT_TOKEN)[0])
@@ -122,7 +130,8 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
     @gr.on(
         triggers=[dataset_radio.change, report_list.change],
         inputs=[dataset_radio, report_list],
-        outputs=[dataset_plot, dataset_table, subset_select, data_review_df, report_analysis])
+        outputs=[dataset_plot, dataset_table, subset_select, data_review_df, report_analysis]
+    )
     def update_single_report_dataset(dataset_name, report_list):
         logger.debug(f'Updating single report dataset: {dataset_name}')
         report_df = get_data_frame(report_list=report_list)
@@ -136,7 +145,8 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
     @gr.on(
         triggers=[subset_select.change],
         inputs=[work_dir, model_name, dataset_radio, subset_select],
-        outputs=[data_review_df, page_number])
+        outputs=[data_review_df, page_number]
+    )
     def update_single_report_subset(work_dir, model_name, dataset_name, subset_name):
         if not subset_name:
             return gr.skip()
@@ -146,7 +156,8 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
     @gr.on(
         triggers=[data_review_df.change, answer_mode_radio.change, score_threshold.change],
         inputs=[data_review_df, answer_mode_radio, score_threshold],
-        outputs=[filtered_review_df, page_number, answer_mode_counts])
+        outputs=[filtered_review_df, page_number, answer_mode_counts]
+    )
     def filter_data(data_review_df, answer_mode, score_threshold):
         if data_review_df is None:
             return None, gr.update(value=1, maximum=1), ''
@@ -172,7 +183,8 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
     @gr.on(
         triggers=[filtered_review_df.change, page_number.change],
         inputs=[filtered_review_df, page_number, score_threshold],
-        outputs=[input_text, generated_text, gold_text, pred_text, score_text, nscore])
+        outputs=[input_text, generated_text, gold_text, pred_text, score_text, nscore]
+    )
     def update_table_components(filtered_df, page_number, score_threshold):
         if filtered_df is None or len(filtered_df) == 0:
             return '', '', '', '', '', ''
@@ -185,10 +197,10 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
         row = filtered_df.iloc[start]
 
         # Process the data for display
-        input_md = process_json_content(row['Input'])
-        generated_md = process_model_prediction(row['Generated'])
-        gold_md = process_model_prediction(row['Gold'])
-        pred_md = convert_markdown_image(process_model_prediction(row['Pred']))
+        input_md = row['Input'] + '\n\n' + process_model_prediction(row['Metadata'])
+        generated_md = convert_markdown_image(row['Generated'])
+        gold_md = convert_markdown_image(row['Gold'])
+        pred_md = process_model_prediction(row['Pred'])
         score_md = process_json_content(row['Score'])
         nscore_val = float(row['NScore']) if not pd.isna(row['NScore']) else 0.0
 

evalscope/app/utils/data_utils.py

@@ -2,14 +2,14 @@
 Data loading and processing utilities for the Evalscope dashboard.
 """
 import glob
-import numpy as np
 import os
 import pandas as pd
 from typing import Any, Dict, List, Union
 
+from evalscope.api.evaluator import CacheManager, ReviewResult
 from evalscope.constants import DataCollection
 from evalscope.report import Report, ReportKey, get_data_frame, get_report_list
-from evalscope.utils.io_utils import OutputsStructure, yaml_to_dict
+from evalscope.utils.io_utils import OutputsStructure, jsonl_to_list, yaml_to_dict
 from evalscope.utils.logger import get_logger
 from ..constants import DATASET_TOKEN, MODEL_TOKEN, REPORT_TOKEN
 
@@ -39,7 +39,8 @@ def scan_for_report_folders(root_path):
                 datasets.append(os.path.splitext(os.path.basename(dataset_item))[0])
             datasets = DATASET_TOKEN.join(datasets)
             reports.append(
-                f'{os.path.basename(folder)}{REPORT_TOKEN}{os.path.basename(model_item)}{MODEL_TOKEN}{datasets}')
+                f'{os.path.basename(folder)}{REPORT_TOKEN}{os.path.basename(model_item)}{MODEL_TOKEN}{datasets}'
+            )
 
     reports = sorted(reports, reverse=True)
     logger.debug(f'reports: {reports}')
@@ -61,7 +62,8 @@ def load_single_report(root_path: str, report_name: str):
     config_files = glob.glob(os.path.join(root_path, prefix, OutputsStructure.CONFIGS_DIR, '*.yaml'))
     if not config_files:
         raise FileNotFoundError(
-            f'No configuration files found in {os.path.join(root_path, prefix, OutputsStructure.CONFIGS_DIR)}')
+            f'No configuration files found in {os.path.join(root_path, prefix, OutputsStructure.CONFIGS_DIR)}'
+        )
     task_cfg_path = config_files[0]
     task_cfg = yaml_to_dict(task_cfg_path)
     return report_list, datasets, task_cfg
@@ -134,31 +136,45 @@ def get_report_analysis(report_list: List[Report], dataset_name: str) -> str:
 
 
 def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subset_name: str):
-    data_path = os.path.join(work_dir, OutputsStructure.REVIEWS_DIR, model_name)
-    subset_name = subset_name.replace('/', '_')  # for collection report
-    review_path = os.path.join(data_path, f'{dataset_name}_{subset_name}.jsonl')
-    logger.debug(f'review_path: {review_path}')
-    origin_df = pd.read_json(review_path, lines=True)
+    # Load review cache
+    outputs = OutputsStructure(work_dir, is_make=False)
+    cache_manager = CacheManager(outputs, model_name, dataset_name)
+    if dataset_name == DataCollection.NAME:
+        review_cache_path = cache_manager.get_review_cache_path('default')
+    else:
+        review_cache_path = cache_manager.get_review_cache_path(subset_name)
+    logger.debug(f'review_path: {review_cache_path}')
+    review_caches = jsonl_to_list(review_cache_path)
 
     ds = []
-    for i, item in origin_df.iterrows():
-        raw_input = item['raw_input']
-        sample_index = item['index']
-        for choice_index, choice in enumerate(item['choices']):
-            raw_pred_answer = choice['message']['content']
-            parsed_gold_answer = choice['review']['gold']
-            parsed_pred_answer = choice['review']['pred']
-            score = choice['review']['result']
-            raw_d = {
-                'Index': f'{sample_index}_{choice_index}',
-                'Input': raw_input,
-                'Generated': raw_pred_answer if raw_pred_answer != parsed_pred_answer else '*Same as Pred*',
-                'Gold': parsed_gold_answer if parsed_gold_answer != raw_input else '*Same as Input*',
-                'Pred': parsed_pred_answer,
-                'Score': score,
-                'NScore': normalize_score(score)
-            }
-            ds.append(raw_d)
+    for cache in review_caches:
+        review_result = ReviewResult.model_validate(cache)
+        sample_score = review_result.sample_score
+
+        if dataset_name == DataCollection.NAME:
+            # Filter subset name
+            collection_info = sample_score.sample_metadata[DataCollection.INFO]
+            sample_dataset_name = collection_info.get('dataset_name', 'default')
+            sample_subset_name = collection_info.get('subset_name', 'default')
+            if f'{sample_dataset_name}/{sample_subset_name}' != subset_name:
+                continue
+
+        score = sample_score.score
+        metadata = sample_score.sample_metadata
+        prediction = score.prediction
+        target = review_result.target
+        extracted_prediction = score.extracted_prediction
+        raw_d = {
+            'Index': str(review_result.index),
+            'Input': review_result.input.replace('\n', '\n\n'),  # for markdown
+            'Metadata': metadata,
+            'Generated': prediction,
+            'Gold': target,
+            'Pred': extracted_prediction if extracted_prediction != prediction else '*Same as Generated*',
+            'Score': score.model_dump(exclude_none=True),
+            'NScore': normalize_score(score.main_value)
+        }
+        ds.append(raw_d)
 
     df_subset = pd.DataFrame(ds)
     return df_subset

evalscope/app/utils/env_utils.py

@@ -0,0 +1,12 @@
+# flake8: noqa
+import os
+
+
+def setup_env(args):
+    compat_dsw_gradio(args)
+
+
+def compat_dsw_gradio(args) -> None:
+    if ('JUPYTER_NAME' in os.environ) and ('dsw-'
+                                           in os.environ['JUPYTER_NAME']) and ('GRADIO_ROOT_PATH' not in os.environ):
+        os.environ['GRADIO_ROOT_PATH'] = f"/{os.environ['JUPYTER_NAME']}/proxy/{args.server_port}"

evalscope/app/utils/text_utils.py

@@ -2,11 +2,9 @@
 Text processing utilities for the Evalscope dashboard.
 """
 import json
-import numpy as np
 import os
-import pandas as pd
 import re
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from evalscope.utils.logger import get_logger
 from ..constants import LATEX_DELIMITERS
@@ -14,15 +12,19 @@ from ..constants import LATEX_DELIMITERS
 logger = get_logger()
 
 
-def convert_markdown_image(text):
-    if not os.path.isfile(text):
-        return text
-    # Convert the image path to a markdown image tag
-    if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
-        text = os.path.abspath(text)
-        image_tag = f'![image](gradio_api/file={text})'
-        logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+def convert_markdown_image(text: str):
+    if text.startswith('data:image'):
+        # Convert base64 image data to a markdown image tag
+        image_tag = f'![image]({text})'
+        logger.debug(f'Converting base64 image data to markdown: {text[:30]}... -> {image_tag[:40]}...')
         return image_tag
+    elif os.path.isfile(text):
+        # Convert the image path to a markdown image tag
+        if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
+            text = os.path.abspath(text)
+            image_tag = f'![image](gradio_api/file={text})'
+            logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+            return image_tag
     return text
 
 
@@ -85,7 +87,7 @@ def process_model_prediction_old(item: Any, max_length: int = 2048) -> str:
     return result
 
 
-def process_model_prediction(item: Any, max_length: int = 32000) -> str:
+def process_model_prediction(item: Any, max_length: Optional[int] = None) -> str:
     if isinstance(item, (dict, list)):
         result = json.dumps(item, ensure_ascii=False, indent=2)
         result = f'```json\n{result}\n```'
@@ -109,8 +111,6 @@ def process_json_content(content: Any) -> str:
     Returns:
         str: The processed content formatted for markdown display.
     """
-    if isinstance(content, (np.bool_, np.int_, np.float_)):
-        content = str(content)
 
     if isinstance(content, str):
         content = {'content': content}

evalscope/app/utils/visualization.py

@@ -47,7 +47,8 @@ def plot_single_report_sunburst(report_list: List[Report]):
         color_continuous_scale='RdYlGn',  # see https://plotly.com/python/builtin-colorscales/
         color_continuous_midpoint=np.average(df[ReportKey.score], weights=df[ReportKey.num]),
         template=PLOTLY_THEME,
-        maxdepth=4)
+        maxdepth=4
+    )
     plot.update_traces(insidetextorientation='radial')
     plot.update_layout(margin=dict(t=10, l=10, r=10, b=10), coloraxis=dict(cmin=0, cmax=1), height=600)
     return plot
@@ -61,7 +62,8 @@ def plot_single_dataset_scores(df: pd.DataFrame):
         y=df[ReportKey.score],
         color=df[ReportKey.subset_name],
         text=df[ReportKey.score],
-        barmode='group')
+        barmode='group'
+    )
 
     width = 0.2 if len(df[ReportKey.subset_name]) <= 3 else None
     plot.update_traces(width=width, texttemplate='%{text:.2f}', textposition='outside')
@@ -82,10 +84,13 @@ def plot_multi_report_radar(df: pd.DataFrame):
                 r=common_group[ReportKey.score],
                 theta=common_group[ReportKey.dataset_name],
                 name=model_name,
-                fill='toself'))
+                fill='toself'
+            )
+        )
 
     fig.update_layout(
         template=PLOTLY_THEME,
         polar=dict(radialaxis=dict(visible=True, range=[0, 1])),
-        margin=dict(t=20, l=20, r=20, b=20))
+        margin=dict(t=20, l=20, r=20, b=20)
+    )
     return fig

evalscope/arguments.py

@@ -1,7 +1,8 @@
+# flake8: noqa: E501
 import argparse
 import json
 
-from evalscope.constants import EvalBackend, EvalStage, EvalType, JudgeStrategy, ModelTask, OutputType
+from evalscope.constants import EvalBackend, EvalType, JudgeStrategy, ModelTask
 
 
 class ParseStrArgsAction(argparse.Action):
@@ -47,7 +48,6 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--model-task', type=str, default=ModelTask.TEXT_GENERATION, choices=[ModelTask.TEXT_GENERATION, ModelTask.IMAGE_GENERATION], help='The model task for model id.')  # noqa: E501
 
     # Template-related arguments
-    parser.add_argument('--template-type', type=str, required=False, help='Deprecated, will be removed in v1.0.0.')
     parser.add_argument('--chat-template', type=str, required=False, help='The custom jinja template for chat generation.')  # noqa: E501
 
     # Dataset-related arguments
@@ -60,30 +60,27 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--generation-config', type=str, action=ParseStrArgsAction, help='The generation config, should be a string.')  # noqa: E501
 
     # Evaluation-related arguments
-    parser.add_argument('--eval-type', type=str, help='The type for evaluating.',
-                        choices=[EvalType.CHECKPOINT, EvalType.CUSTOM, EvalType.SERVICE])
+    parser.add_argument('--eval-type', type=str, help='The type for evaluating.')
     parser.add_argument('--eval-backend', type=str, help='The evaluation backend to use.',
                         choices=[EvalBackend.NATIVE, EvalBackend.OPEN_COMPASS, EvalBackend.VLM_EVAL_KIT, EvalBackend.RAG_EVAL])  # noqa: E501
     parser.add_argument('--eval-config', type=str, required=False, help='The eval task config file path for evaluation backend.')  # noqa: E501
-    parser.add_argument('--stage', type=str, default='all', help='The stage of evaluation pipeline.',
-                        choices=[EvalStage.ALL, EvalStage.INFER, EvalStage.REVIEW])
-    parser.add_argument('--limit', type=float, default=None, help='Max evaluation samples num for each subset.')
     parser.add_argument('--eval-batch-size', type=int, default=1, help='The batch size for evaluation.')
+    parser.add_argument('--limit', type=float, default=None, help='Max evaluation samples num for each subset.')
+    parser.add_argument('--repeats', type=int, default=1, help='Number of times to repeat the dataset items for k-metrics.')  # noqa: E501
 
     # Cache and working directory arguments
-    parser.add_argument('--mem-cache', action='store_true', default=False, help='Deprecated, will be removed in v1.0.0.')  # noqa: E501
     parser.add_argument('--use-cache', type=str, help='Path to reuse the cached results.')
+    parser.add_argument('--rerun-review', action='store_true', default=False, help='Rerun the review process when use_cache.')
     parser.add_argument('--work-dir', type=str, help='The root cache dir.')
 
     # Debug and runtime mode arguments
     parser.add_argument('--ignore-errors', action='store_true', default=False, help='Ignore errors during evaluation.')
     parser.add_argument('--debug', action='store_true', default=False, help='Debug mode, will print information for debugging.')  # noqa: E501
-    parser.add_argument('--dry-run', action='store_true', default=False, help='Dry run in single processing mode.')
     parser.add_argument('--seed', type=int, default=42, help='Random seed for reproducibility.')
     parser.add_argument('--api-key', type=str, default='EMPTY', help='The API key for the remote API model.')
     parser.add_argument('--api-url', type=str, default=None, help='The API url for the remote API model.')
     parser.add_argument('--timeout', type=float, default=None, help='The timeout for the remote API model.')
-    parser.add_argument('--stream', action='store_true', default=False, help='Stream mode.')  # noqa: E501
+    parser.add_argument('--stream', action='store_true', default=None, help='Stream mode.')  # noqa: E501
 
     # LLMJudge arguments
     parser.add_argument('--judge-strategy', type=str, default=JudgeStrategy.AUTO, help='The judge strategy.')

evalscope/backend/opencompass/api_meta_template.py

@@ -49,7 +49,8 @@ register_template(
         reserved_roles=[
             dict(role='SYSTEM', api_role='SYSTEM'),
         ],
-    ))
+    )
+)
 
 if __name__ == '__main__':
     res = MetaTemplateType.get_template_name_list()

evalscope/backend/opencompass/backend_manager.py

@@ -47,7 +47,6 @@ class OpenCompassBackendManager(BackendManager):
                     datasets: list, the datasets.
                     models: list, the models.
                     work_dir (Optional): str, the working directory. Default to None, which means the current directory.
-                    dry_run (Optional): bool, the dry-run flag. Default to False.
                     debug (Optional): bool, the debug flag. Default to False.
                     reuse (Optional): str, reuse previous outputs & results. Default to None.
                     generation_kwargs (Optional): dict, the generation config. Default to {}.
@@ -140,7 +139,6 @@ class OpenCompassBackendManager(BackendManager):
             cmd_str = f'python -m run_oc ' \
                       f'--models {" ".join(self.args.models)} ' \
                       f'--datasets {" ".join(self.args.datasets)} ' \
-                      f'{self.get_restore_arg("dry-run", self.args.dry_run)} ' \
                       f'{self.get_arg_with_default("work-dir", self.args.work_dir)}'
 
         elif cmd_mode == CmdMode.SCRIPT:
@@ -182,8 +180,10 @@ class OpenCompassBackendManager(BackendManager):
             else:
                 valid_dataset_names, invalid_dataset_names = get_valid_list(dataset_names, dataset_names_all)
                 if len(invalid_dataset_names) > 0:
-                    logger.error(f'Invalid datasets: {invalid_dataset_names}, '
-                                 f'refer to the following list to get proper dataset name: {dataset_names_all}')
+                    logger.error(
+                        f'Invalid datasets: {invalid_dataset_names}, '
+                        f'refer to the following list to get proper dataset name: {dataset_names_all}'
+                    )
                 assert len(valid_dataset_names) > 0, f'No valid datasets. ' \
                                                      f'To get the valid datasets, please refer to {dataset_names_all}'
 
@@ -252,7 +252,8 @@ if __name__ == '__main__':
                 'openai_api_base': 'http://127.0.0.1:8000/v1/chat/completions'
             }],
             'limit': 5
-        })
+        }
+    )
     all_datasets = OpenCompassBackendManager.list_datasets()
     print(f'all_datasets: {all_datasets}')
     oc_backend_manager.run()

evalscope/backend/rag_eval/clip_benchmark/dataset_builder.py

@@ -100,16 +100,16 @@ class DatasetWrapper(TorchDataset):
 
 def get_dataset_default_task(dataset):
     if dataset in (
-            'custom',
-            'muge',
-            'flickr30k',
-            'flickr8k',
-            'mscoco_captions',
-            'mscoco_captions2017',
-            'multilingual_mscoco_captions',
-            'flickr30k-200',
-            'crossmodal3600',
-            'xtd200',
+        'custom',
+        'muge',
+        'flickr30k',
+        'flickr8k',
+        'mscoco_captions',
+        'mscoco_captions2017',
+        'multilingual_mscoco_captions',
+        'flickr30k-200',
+        'crossmodal3600',
+        'xtd200',
     ):
         return 'zeroshot_retrieval'
     else:

evalscope/backend/rag_eval/clip_benchmark/task_template.py

@@ -4,8 +4,11 @@ import torch
 from itertools import product
 
 from evalscope.backend.rag_eval.clip_benchmark.arguments import Arguments
-from evalscope.backend.rag_eval.clip_benchmark.dataset_builder import (build_dataset, get_dataloader,
-                                                                       get_dataset_default_task)
+from evalscope.backend.rag_eval.clip_benchmark.dataset_builder import (
+    build_dataset,
+    get_dataloader,
+    get_dataset_default_task,
+)
 from evalscope.backend.rag_eval.clip_benchmark.tasks import image_caption, zeroshot_classification, zeroshot_retrieval
 from evalscope.backend.rag_eval.utils.clip import VisionModel
 from evalscope.utils.logger import get_logger
@@ -66,8 +69,9 @@ def evaluate(args: Arguments):
             if verbose:
                 logger.info(f'Zero-shot templates: {zeroshot_templates}')
             classnames = dataset.classes if hasattr(dataset, 'classes') else None
-            assert (zeroshot_templates is not None
-                    and classnames is not None), 'Dataset does not support classification'
+            assert (
+                zeroshot_templates is not None and classnames is not None
+            ), 'Dataset does not support classification'
             metrics = zeroshot_classification.evaluate(
                 model,
                 dataloader,

evalscope/backend/rag_eval/ragas/task_template.py

@@ -34,7 +34,8 @@ def rag_eval(args: EvaluationArguments, ) -> None:
             target_lang=args.language,
             llm=LangchainLLMWrapper(llm),
             adapt_instruction=True,
-        ))
+        )
+    )
     # load dataset
     dataset = Dataset.from_json(args.testset_file)