evalscope 0.15.1__py3-none-any.whl → 0.16.1__py3-none-any.whl

evalscope/app/__init__.py

@@ -0,0 +1,28 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from typing import TYPE_CHECKING
+
+from evalscope.utils.import_utils import _LazyModule
+
+if TYPE_CHECKING:
+    from .app import create_app
+    from .arguments import add_argument
+
+else:
+    _import_structure = {
+        'app': [
+            'create_app',
+        ],
+        'arguments': [
+            'add_argument',
+        ],
+    }
+
+    import sys
+
+    sys.modules[__name__] = _LazyModule(
+        __name__,
+        globals()['__file__'],
+        _import_structure,
+        module_spec=__spec__,
+        extra_objects={},
+    )

evalscope/{report → app}/app.py

@@ -11,35 +11,15 @@ from dataclasses import dataclass
 from typing import Any, List, Union
 
 from evalscope.constants import DataCollection
-from evalscope.report import Report, ReportKey, add_argument, get_data_frame, get_report_list
+from evalscope.report import Report, ReportKey, get_data_frame, get_report_list
 from evalscope.utils.io_utils import OutputsStructure, yaml_to_dict
 from evalscope.utils.logger import configure_logging, get_logger
 from evalscope.version import __version__
+from .arguments import add_argument
+from .constants import DATASET_TOKEN, LATEX_DELIMITERS, MODEL_TOKEN, PLOTLY_THEME, REPORT_TOKEN
 
 logger = get_logger()
 
-PLOTLY_THEME = 'plotly_dark'
-REPORT_TOKEN = '@@'
-MODEL_TOKEN = '::'
-DATASET_TOKEN = ', '
-LATEX_DELIMITERS = [{
-    'left': '$$',
-    'right': '$$',
-    'display': True
-}, {
-    'left': '$',
-    'right': '$',
-    'display': False
-}, {
-    'left': '\\(',
-    'right': '\\)',
-    'display': False
-}, {
-    'left': '\\[',
-    'right': '\\]',
-    'display': True
-}]
-
 
 def scan_for_report_folders(root_path):
     """Scan for folders containing reports subdirectories"""
@@ -185,6 +165,13 @@ def get_single_dataset_df(df: pd.DataFrame, dataset_name: str):
     return df, styler
 
 
+def get_report_analysis(report_list: List[Report], dataset_name: str) -> str:
+    for report in report_list:
+        if report.dataset_name == dataset_name:
+            return report.analysis
+    return 'N/A'
+
+
 def plot_single_dataset_scores(df: pd.DataFrame):
     # TODO: add metric radio and relace category name
     plot = px.bar(
@@ -223,6 +210,33 @@ def plot_multi_report_radar(df: pd.DataFrame):
     return fig
 
 
+def convert_markdown_image(text):
+    if not os.path.isfile(text):
+        return text
+    # Convert the image path to a markdown image tag
+    if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
+        text = os.path.abspath(text)
+        image_tag = f'![image](gradio_api/file={text})'
+        logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+        return image_tag
+    return text
+
+
+def convert_html_tags(text):
+    # match begin label
+    text = re.sub(r'<(\w+)>', r'[\1]', text)
+    # match end label
+    text = re.sub(r'</(\w+)>', r'[/\1]', text)
+    return text
+
+
+def process_string(string: str, max_length: int = 2048) -> str:
+    string = convert_html_tags(string)  # for display labels e.g.
+    if max_length and len(string) > max_length:
+        return f'{string[:max_length // 2]}......{string[-max_length // 2:]}'
+    return string
+
+
 def dict_to_markdown(data) -> str:
     markdown_lines = []
 
@@ -230,55 +244,41 @@ def dict_to_markdown(data) -> str:
         bold_key = f'**{key}**'
 
         if isinstance(value, list):
-            value_str = '\n' + '\n'.join([f'  - {item}' for item in value])
+            value_str = '\n' + '\n'.join([f'- {process_model_prediction(item, max_length=None)}' for item in value])
         elif isinstance(value, dict):
             value_str = dict_to_markdown(value)
         else:
             value_str = str(value)
 
-        value_str = process_string(value_str)
-        markdown_line = f'{bold_key}: {value_str}'
+        value_str = process_string(value_str, max_length=None)  # Convert HTML tags but don't truncate
+        markdown_line = f'{bold_key}:\n{value_str}'
         markdown_lines.append(markdown_line)
 
     return '\n\n'.join(markdown_lines)
 
 
-def convert_html_tags(text):
-    # match begin label
-    text = re.sub(r'<(\w+)>', r'[\1]', text)
-    # match end label
-    text = re.sub(r'</(\w+)>', r'[/\1]', text)
-    return text
+def process_model_prediction(item: Any, max_length: int = 2048) -> str:
+    """
+    Process model prediction output into a formatted string.
 
+    Args:
+        item: The item to process. Can be a string, list, or dictionary.
+        max_length: The maximum length of the output string.
 
-def convert_markdown_image(text):
-    if not os.path.isfile(text):
-        return text
-    # Convert the image path to a markdown image tag
-    if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
-        text = os.path.abspath(text)
-        image_tag = f'![image](gradio_api/file={text})'
-        logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
-        return image_tag
-    return text
-
-
-def process_string(string: str, max_length: int = 2048) -> str:
-    string = convert_html_tags(string)  # for display labels e.g. `<think>`
-    if len(string) > max_length:
-        return f'{string[:max_length // 2]}......{string[-max_length // 2:]}'
-    return string
-
-
-def process_model_prediction(item: Any):
+    Returns:
+        A formatted string representation of the input.
+    """
     if isinstance(item, dict):
-        res = dict_to_markdown(item)
-        return process_string(res)
+        result = dict_to_markdown(item)
     elif isinstance(item, list):
-        res = '\n'.join([process_model_prediction(item) for item in item])
-        return process_string(res)
+        result = '\n'.join([f'- {process_model_prediction(i, max_length=None)}' for i in item])
     else:
-        return process_string(str(item))
+        result = str(item)
+
+    # Apply HTML tag conversion and truncation only at the final output
+    if max_length is not None:
+        return process_string(result, max_length)
+    return result
 
 
 def normalize_score(score):
@@ -443,6 +443,10 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
             'zh': '数据集分数',
             'en': 'Dataset Scores'
         },
+        'report_analysis': {
+            'zh': '报告智能分析',
+            'en': 'Report Intelligent Analysis'
+        },
         'dataset_scores_table': {
             'zh': '数据集分数表',
             'en': 'Dataset Scores Table'
@@ -498,6 +502,9 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
     with gr.Tab(locale_dict['dataset_details'][lang]):
         dataset_radio = gr.Radio(
             label=locale_dict['select_dataset'][lang], choices=[], show_label=True, interactive=True)
+        # show dataset details
+        with gr.Accordion(locale_dict['report_analysis'][lang], open=True):
+            report_analysis = gr.Markdown(value='N/A', show_copy_button=True)
         gr.Markdown(f'### {locale_dict["dataset_scores"][lang]}')
         dataset_plot = gr.Plot(value=None, scale=1, label=locale_dict['dataset_scores'][lang])
         gr.Markdown(f'### {locale_dict["dataset_scores_table"][lang]}')
@@ -573,15 +580,16 @@ def create_single_model_tab(sidebar: SidebarComponents, lang: str):
     @gr.on(
         triggers=[dataset_radio.change, report_list.change],
         inputs=[dataset_radio, report_list],
-        outputs=[dataset_plot, dataset_table, subset_select, data_review_df])
+        outputs=[dataset_plot, dataset_table, subset_select, data_review_df, report_analysis])
     def update_single_report_dataset(dataset_name, report_list):
         logger.debug(f'Updating single report dataset: {dataset_name}')
         report_df = get_data_frame(report_list)
+        analysis = get_report_analysis(report_list, dataset_name)
         data_score_df, styler = get_single_dataset_df(report_df, dataset_name)
         data_score_plot = plot_single_dataset_scores(data_score_df)
         subsets = data_score_df[ReportKey.subset_name].unique().tolist()
         logger.debug(f'subsets: {subsets}')
-        return data_score_plot, styler, gr.update(choices=subsets, value=None), None
+        return data_score_plot, styler, gr.update(choices=subsets, value=None), None, analysis
 
     @gr.on(
         triggers=[subset_select.change],

evalscope/app/constants.py

@@ -0,0 +1,21 @@
+PLOTLY_THEME = 'plotly_dark'
+REPORT_TOKEN = '@@'
+MODEL_TOKEN = '::'
+DATASET_TOKEN = ', '
+LATEX_DELIMITERS = [{
+    'left': '$$',
+    'right': '$$',
+    'display': True
+}, {
+    'left': '$',
+    'right': '$',
+    'display': False
+}, {
+    'left': '\\(',
+    'right': '\\)',
+    'display': False
+}, {
+    'left': '\\[',
+    'right': '\\]',
+    'display': True
+}]

evalscope/arguments.py

@@ -9,6 +9,15 @@ class ParseStrArgsAction(argparse.Action):
     def __call__(self, parser, namespace, values, option_string=None):
         assert isinstance(values, str), 'args should be a string.'
 
+        # try json load first
+        try:
+            arg_dict = json.loads(values)
+            setattr(namespace, self.dest, arg_dict)
+            return
+        except (json.JSONDecodeError, ValueError):
+            pass
+
+        # If JSON load fails, fall back to parsing as key=value pairs
         arg_dict = {}
         for arg in values.strip().split(','):
             key, value = map(str.strip, arg.split('=', 1))  # Use maxsplit=1 to handle multiple '='
@@ -58,7 +67,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--eval-config', type=str, required=False, help='The eval task config file path for evaluation backend.')  # noqa: E501
     parser.add_argument('--stage', type=str, default='all', help='The stage of evaluation pipeline.',
                         choices=[EvalStage.ALL, EvalStage.INFER, EvalStage.REVIEW])
-    parser.add_argument('--limit', type=int, default=None, help='Max evaluation samples num for each subset.')
+    parser.add_argument('--limit', type=float, default=None, help='Max evaluation samples num for each subset.')
     parser.add_argument('--eval-batch-size', type=int, default=1, help='The batch size for evaluation.')
 
     # Cache and working directory arguments
@@ -67,6 +76,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--work-dir', type=str, help='The root cache dir.')
 
     # Debug and runtime mode arguments
+    parser.add_argument('--ignore-errors', action='store_true', default=False, help='Ignore errors during evaluation.')
     parser.add_argument('--debug', action='store_true', default=False, help='Debug mode, will print information for debugging.')  # noqa: E501
     parser.add_argument('--dry-run', action='store_true', default=False, help='Dry run in single processing mode.')
     parser.add_argument('--seed', type=int, default=42, help='Random seed for reproducibility.')
@@ -79,6 +89,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--judge-strategy', type=str, default=JudgeStrategy.AUTO, help='The judge strategy.')
     parser.add_argument('--judge-model-args', type=json.loads, default='{}', help='The judge model args, should be a json string.')  # noqa: E501
     parser.add_argument('--judge-worker-num', type=int, default=1, help='The number of workers for the judge model.')
+    parser.add_argument('--analysis-report', action='store_true', default=False, help='Generate analysis report for the evaluation results using judge model.')  # noqa: E501
     # yapf: enable
 
 

evalscope/backend/opencompass/backend_manager.py

@@ -1,4 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+import os
 import subprocess
 import tempfile
 from dataclasses import asdict
@@ -204,7 +205,7 @@ class OpenCompassBackendManager(BackendManager):
                     model_d['meta_template'] = get_template(model_d['meta_template'])
 
                 # set the 'abbr' as the 'path' if 'abbr' is not specified
-                model_d['abbr'] = model_d['path']
+                model_d['abbr'] = os.path.basename(model_d['path'])
 
                 model_config = ApiModelConfig(**model_d)
                 models.append(asdict(model_config))

evalscope/backend/rag_eval/cmteb/arguments.py

@@ -11,7 +11,9 @@ class ModelArguments:
     pooling_mode: Optional[str] = None
     max_seq_length: int = 512  # max sequence length
     # prompt for llm based model
-    prompt: str = ''
+    prompt: Optional[str] = None
+    # prompts dictionary for different tasks, if prompt is not set
+    prompts: Optional[Dict[str, str]] = None
     # model kwargs
     model_kwargs: dict = field(default_factory=dict)
     # config kwargs
@@ -33,6 +35,7 @@ class ModelArguments:
             'pooling_mode': self.pooling_mode,
             'max_seq_length': self.max_seq_length,
             'prompt': self.prompt,
+            'prompts': self.prompts,
             'model_kwargs': self.model_kwargs,
             'config_kwargs': self.config_kwargs,
             'encode_kwargs': self.encode_kwargs,

evalscope/backend/rag_eval/cmteb/task_template.py

@@ -1,6 +1,6 @@
 import mteb
 import os
-from mteb.task_selection import results_to_dataframe
+from tabulate import tabulate
 
 from evalscope.backend.rag_eval import EmbeddingModel, cmteb
 from evalscope.utils.logger import get_logger
@@ -12,14 +12,27 @@ def show_results(output_folder, model, results):
     model_name = model.mteb_model_meta.model_name_as_path()
     revision = model.mteb_model_meta.revision
 
-    results_df = results_to_dataframe({model_name: {revision: results}})
+    data = []
+    for model_res in results:
+        main_res = model_res.only_main_score()
+        for split, score in main_res.scores.items():
+            for sub_score in score:
+                data.append({
+                    'Model': model_name.replace('eval__', ''),
+                    'Revision': revision,
+                    'Task Type': main_res.task_type,
+                    'Task': main_res.task_name,
+                    'Split': split,
+                    'Subset': sub_score['hf_subset'],
+                    'Main Score': sub_score['main_score'],
+                })
 
     save_path = os.path.join(
         output_folder,
         model_name,
         revision,
     )
-    logger.info(f'Evaluation results:\n{results_df.to_markdown()}')
+    logger.info(f'Evaluation results:\n{tabulate(data, headers="keys", tablefmt="grid")}')
     logger.info(f'Evaluation results saved in {os.path.abspath(save_path)}')
 
 
@@ -34,6 +47,7 @@ def one_stage_eval(
     tasks = cmteb.TaskBase.get_tasks(task_names=eval_args['tasks'], dataset_path=custom_dataset_path)
     evaluation = mteb.MTEB(tasks=tasks)
 
+    eval_args['encode_kwargs'] = model_args.get('encode_kwargs', {})
     # run evaluation
     results = evaluation.run(model, **eval_args)
 
@@ -66,6 +80,7 @@ def two_stage_eval(
             overwrite_results=True,
             hub=eval_args['hub'],
             limits=eval_args['limits'],
+            encode_kwargs=model1_args.get('encode_kwargs', {}),
         )
         # stage 2: run cross encoder
         results = evaluation.run(
@@ -77,6 +92,7 @@ def two_stage_eval(
             overwrite_results=True,
             hub=eval_args['hub'],
             limits=eval_args['limits'],
+            encode_kwargs=model2_args.get('encode_kwargs', {}),
         )
 
         # save and log results

evalscope/backend/rag_eval/cmteb/tasks/CustomTask.py

@@ -9,7 +9,6 @@ class CustomRetrieval(AbsTaskRetrieval):
     ignore_identical_ids: bool = True
 
     def __init__(self, dataset_path: Optional[str] = 'custom_eval/text/retrieval', **kwargs):
-        super().__init__(**kwargs)
         self.metadata = TaskMetadata(
             name='CustomRetrieval',
             description='CustomRetrieval Task',
@@ -34,6 +33,7 @@ class CustomRetrieval(AbsTaskRetrieval):
             bibtex_citation='',
             descriptive_stats={},
         )
+        super().__init__(**kwargs)
 
     def load_data(self, **kwargs):
         if self.data_loaded:

evalscope/backend/rag_eval/utils/embedding.py

@@ -2,6 +2,7 @@ import os
 import torch
 from langchain_core.embeddings import Embeddings
 from langchain_openai.embeddings import OpenAIEmbeddings
+from mteb.encoder_interface import PromptType
 from sentence_transformers import models
 from sentence_transformers.cross_encoder import CrossEncoder
 from sentence_transformers.SentenceTransformer import SentenceTransformer
@@ -12,6 +13,7 @@ from typing import Dict, List, Optional, Union
 from evalscope.backend.rag_eval.utils.tools import download_model
 from evalscope.constants import HubType
 from evalscope.utils.logger import get_logger
+from evalscope.utils.utils import get_supported_params
 
 logger = get_logger()
 
@@ -22,14 +24,14 @@ class BaseModel(Embeddings):
         self,
         model_name_or_path: str = '',
         max_seq_length: int = 512,
-        prompt: str = '',
+        prompt: Optional[str] = None,
+        prompts: Optional[Dict[str, str]] = None,
         revision: Optional[str] = 'master',
         **kwargs,
     ):
         self.model_name_or_path = model_name_or_path
         self.max_seq_length = max_seq_length
         self.model_kwargs = kwargs.pop('model_kwargs', {})
-        self.model_kwargs['trust_remote_code'] = True
 
         self.config_kwargs = kwargs.pop('config_kwargs', {})
         self.config_kwargs['trust_remote_code'] = True
@@ -38,7 +40,9 @@ class BaseModel(Embeddings):
         self.encode_kwargs['convert_to_tensor'] = True
 
         self.prompt = prompt
+        self.prompts = prompts if prompts else {}
         self.revision = revision
+        self.framework = ['PyTorch']
 
     @property
     def mteb_model_meta(self):
@@ -46,10 +50,22 @@ class BaseModel(Embeddings):
         from mteb import ModelMeta
 
         return ModelMeta(
-            name=os.path.basename(self.model_name_or_path),
+            name='eval/' + os.path.basename(self.model_name_or_path),  # Ensure the name contains a slash
             revision=self.revision,
             languages=None,
             release_date=None,
+            n_parameters=None,
+            memory_usage_mb=None,
+            max_tokens=None,
+            embed_dim=None,
+            license=None,
+            open_weights=None,
+            public_training_code=None,
+            public_training_data=None,
+            similarity_fn_name=None,
+            use_instructions=None,
+            training_datasets=None,
+            framework=self.framework,
         )
 
     def embed_documents(self, texts: List[str]) -> List[List[float]]:
@@ -61,7 +77,7 @@ class BaseModel(Embeddings):
         Returns:
             List of embeddings.
         """
-        return self.encode_corpus(texts).tolist()
+        return self.encode(texts).tolist()
 
     def embed_query(self, text: str) -> List[float]:
         """Embed query text. Compact langchain.
@@ -72,19 +88,17 @@ class BaseModel(Embeddings):
         Returns:
             Embedding.
         """
-        return self.encode_queries(text).tolist()
+        return self.encode(text).tolist()
 
     def encode(self, texts: Union[str, List[str]], **kwargs) -> List[List[float]]:
         """Embed text."""
         raise NotImplementedError
 
-    def encode_queries(self, queries: List[str], **kwargs) -> list[torch.Tensor]:
-        """Embed query text. Compact mteb."""
-        raise NotImplementedError
-
-    def encode_corpus(self, corpus: Union[List[str], List[Dict[str, str]]], **kwargs) -> list[torch.Tensor]:
-        """Embed search docs . Compact mteb."""
-        raise NotImplementedError
+    def get_prompt(self, task_name: str) -> Optional[str]:
+        """Get prompt for the given task name."""
+        if self.prompt:
+            return self.prompt
+        return self.prompts.get(task_name, None)
 
 
 class SentenceTransformerModel(BaseModel):
@@ -92,6 +106,9 @@ class SentenceTransformerModel(BaseModel):
     def __init__(self, model_name_or_path: str, pooling_mode: Optional[str] = None, **kwargs):
         super().__init__(model_name_or_path, **kwargs)
 
+        self.framework = ['Sentence Transformers', 'PyTorch']
+
+        self.model_kwargs['trust_remote_code'] = True
         if not pooling_mode:
             self.model = SentenceTransformer(
                 self.model_name_or_path,
@@ -112,36 +129,47 @@ class SentenceTransformerModel(BaseModel):
 
         self.model.max_seq_length = self.max_seq_length
 
-    def encode(self, texts: Union[str, List[str]], prompt=None, **kwargs) -> List[torch.Tensor]:
-        kwargs.pop('prompt_name', '')  # remove prompt name, use prompt
+        self.supported_encode_params = get_supported_params(self.model.encode)
+
+    def encode(self, texts: Union[str, List[str]], **kwargs) -> List[torch.Tensor]:
+        # pop unused kwargs
+        extra_params = {}
+        for key in list(kwargs.keys()):
+            if key not in self.supported_encode_params:
+                extra_params[key] = kwargs.pop(key)
         self.encode_kwargs.update(kwargs)
 
+        # set prompt if provided
+        prompt = None
+        prompt_type = extra_params.pop('prompt_type', '')
+        task_name = extra_params.pop('task_name', '')
+        if prompt_type and prompt_type == PromptType.query:
+            prompt = self.get_prompt(task_name)
+
         embeddings = self.model.encode(texts, prompt=prompt, **self.encode_kwargs)
         assert isinstance(embeddings, Tensor)
         return embeddings.cpu().detach()
 
-    def encode_queries(self, queries, **kwargs):
-        return self.encode(queries, prompt=self.prompt)
-
-    def encode_corpus(self, corpus, **kwargs):
-        if isinstance(corpus[0], dict):
-            input_texts = ['{} {}'.format(doc.get('title', ''), doc['text']).strip() for doc in corpus]
-        else:
-            input_texts = corpus
-        return self.encode(input_texts)
-
 
 class CrossEncoderModel(BaseModel):
 
     def __init__(self, model_name_or_path: str, **kwargs):
         super().__init__(model_name_or_path, **kwargs)
+
+        self.framework = ['Sentence Transformers', 'PyTorch']
+
         self.model = CrossEncoder(
             self.model_name_or_path,
             trust_remote_code=True,
             max_length=self.max_seq_length,
+            automodel_args=self.model_kwargs,
         )
+        self.supported_encode_params = get_supported_params(self.model.predict)
 
     def predict(self, sentences: List[List[str]], **kwargs) -> Tensor:
+        for key in list(kwargs.keys()):
+            if key not in self.supported_encode_params:
+                kwargs.pop(key)
         self.encode_kwargs.update(kwargs)
 
         if len(sentences[0]) == 3:  # Note: For mteb retrieval task
@@ -163,6 +191,7 @@ class APIEmbeddingModel(BaseModel):
         self.openai_api_base = kwargs.get('api_base')
         self.openai_api_key = kwargs.get('api_key')
         self.dimensions = kwargs.get('dimensions')
+        self.framework = ['API']
 
         self.model = OpenAIEmbeddings(
             model=self.model_name,
@@ -175,26 +204,37 @@ class APIEmbeddingModel(BaseModel):
 
         self.batch_size = self.encode_kwargs.get('batch_size', 10)
 
+        self.supported_encode_params = get_supported_params(self.model.embed_documents)
+
     def encode(self, texts: Union[str, List[str]], **kwargs) -> Tensor:
+        # pop unused kwargs
+        extra_params = {}
+        for key in list(kwargs.keys()):
+            if key not in self.supported_encode_params:
+                extra_params[key] = kwargs.pop(key)
+        self.encode_kwargs.update(kwargs)
+
+        # set prompt if provided
+        prompt = None
+        prompt_type = extra_params.pop('prompt_type', '')
+        task_name = extra_params.pop('task_name', '')
+        if prompt_type and prompt_type == PromptType.query:
+            prompt = self.get_prompt(task_name)
+
         if isinstance(texts, str):
             texts = [texts]
 
         embeddings: List[List[float]] = []
         for i in tqdm(range(0, len(texts), self.batch_size)):
-            response = self.model.embed_documents(texts[i:i + self.batch_size], chunk_size=self.batch_size)
+            # set prompt if provided
+            if prompt is not None:
+                batch_texts = [prompt + text for text in texts[i:i + self.batch_size]]
+            else:
+                batch_texts = texts[i:i + self.batch_size]
+            response = self.model.embed_documents(batch_texts, chunk_size=self.batch_size)
             embeddings.extend(response)
         return torch.tensor(embeddings)
 
-    def encode_queries(self, queries, **kwargs):
-        return self.encode(queries, **kwargs)
-
-    def encode_corpus(self, corpus, **kwargs):
-        if isinstance(corpus[0], dict):
-            input_texts = ['{} {}'.format(doc.get('title', ''), doc['text']).strip() for doc in corpus]
-        else:
-            input_texts = corpus
-        return self.encode(input_texts, **kwargs)
-
 
 class EmbeddingModel:
     """Custom embeddings"""

evalscope/backend/rag_eval/utils/llm.py

@@ -52,7 +52,7 @@ class LocalLLM(BaseLLM):
         """Run the LLM on the given input."""
         infer_cfg = {'stop': stop}
 
-        response, _ = self.model._model_generate([prompt], infer_cfg=infer_cfg)
+        response, _ = self.model.predict([{'data': [prompt]}], infer_cfg=infer_cfg)
         return response[0][0]
 
     @property

evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py

@@ -96,12 +96,6 @@ class AlpacaEvalAdapter(DataAdapter):
             return None
 
     def compute_metric(self, review_res_list: List[bool], **kwargs) -> List[dict]:
-        """
-        compute weighted mean of the bleu score of all samples
-
-        Args:
-            review_res_list: [{'is_correct': 1, 'is_incorrect': 0, 'is_not_attempted': 0}, ...]
-        """
         # zip dict answers
         res_list = [res for res in review_res_list if res is not None]
 

evalscope/benchmarks/benchmark.py

@@ -28,6 +28,7 @@ class BenchmarkMeta:
     system_prompt: Optional[str] = None
     query_template: Optional[str] = None
     pretty_name: Optional[str] = None
+    description: Optional[str] = None
     filters: Optional[OrderedDict] = None
     extra_params: Optional[Dict] = field(default_factory=dict)
 

evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py

@@ -148,6 +148,7 @@ class ChineseSimpleQAAdapter(DataAdapter):
             'is_correct': 1 if res == 'A' else 0,
             'is_incorrect': 1 if res == 'B' else 0,
             'is_not_attempted': 1 if res == 'C' else 0,
+            'judge_response': grading_response,
         }
 
     def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]: