evalscope 0.10.0__py3-none-any.whl → 0.11.0__py3-none-any.whl

evalscope/benchmarks/math_500/math_500_adapter.py

@@ -0,0 +1,49 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
+from evalscope.models import ChatGenerationModelAdapter
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='math_500',
+    dataset_id='AI-ModelScope/MATH-500',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['default'],
+    metric_list=['AveragePass@1'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+)
+class Math500Adapter(DataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate the prompt for the model input.
+        """
+        problem = input_d['problem']
+        full_prompt = self.prompt_template.format(query=problem)
+
+        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Extract the gold answer from the input dict.
+        return strip_answer_string(input_d['answer'])
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+        """
+        Parse the model output to get the answer. Could be the best choice index.
+        """
+        # Note: Use same extraction method for both of checkpoint/service/custom
+        result = strip_answer_string(extract_answer(result))
+        return result
+
+    def match(self, gold: str, pred: str) -> float:
+        return math_equal(pred, gold)

evalscope/benchmarks/mmlu/mmlu_adapter.py

@@ -4,17 +4,15 @@ import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy, exact_match
+from evalscope.metrics import exact_match
 from evalscope.models import MultiChoiceModelAdapter
-from evalscope.utils import ResponseParser, normalize_score
+from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
 
 logger = get_logger()
 
-DATASET_ID = 'modelscope/mmlu'
-
 SUBSET_LIST = [
     'high_school_european_history',
     'business_ethics',
@@ -141,11 +139,11 @@ SUBJECT_MAPPING = {
     dataset_id='modelscope/mmlu',
     model_adapter=MultiChoiceModelAdapter,
     subset_list=SUBSET_LIST,
-    metric_list=[AverageAccuracy],
+    metric_list=['AverageAccuracy'],
     few_shot_num=5,
     train_split='train',
     eval_split='test',
-    prompt_template='',
+    prompt_template='The following are multiple choice questions (with answers) about {subset_name}. \n{query}',
 )
 class MMLUAdapter(DataAdapter):
 
@@ -221,17 +219,15 @@ class MMLUAdapter(DataAdapter):
             {'data': [full_prompt], 'multi_choices': self.choices}
 
         """
-        prompt = 'The following are multiple choice questions (with answers) about {}.\n\n'.format(
-            self._format_subject(subset_name))
         few_shot_prompts = [self._generate_prompt(input_d=sample, include_answer=True) for sample in few_shot_list]
 
         context: str = '\n'.join(few_shot_prompts) + '\n'
         context += self._generate_prompt(input_d=input_d, include_answer=False)
-        context = prompt + context
+        query = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
+        full_prompt = self.prompt_template.format(subset_name=self._format_subject(subset_name), query=query)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
+        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -3,22 +3,27 @@ from typing import Any, Dict
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import AnswerKeys, EvalType
-from evalscope.metrics import AverageAccuracy, exact_match
+from evalscope.metrics import exact_match
 from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.utils import ResponseParser
 
+SUBSET_LIST = [
+    'computer science', 'math', 'chemistry', 'engineering', 'law', 'biology', 'health', 'physics', 'business',
+    'philosophy', 'economics', 'other', 'psychology', 'history'
+]
+
 
 @Benchmark.register(
     name='mmlu_pro',
     dataset_id='modelscope/mmlu-pro',
     model_adapter=ChatGenerationModelAdapter,
-    subset_list=['default'],
-    metric_list=[AverageAccuracy],
+    subset_list=SUBSET_LIST,
+    metric_list=['AverageAccuracy'],
     few_shot_num=5,
     train_split='validation',
     eval_split='test',
     prompt_template=
-    'You are an knowledge expert, you are supposed to answer the multi-choice question to derive your final answer as `The answer is ...`.',  # noqa: E501
+    'The following are multiple choice questions (with answers) about {subset_name}. Think step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.\n{query}',  # noqa: E501
 )
 class MMLUProAdapter(DataAdapter):
 
@@ -26,10 +31,11 @@ class MMLUProAdapter(DataAdapter):
         super().__init__(**kwargs)
 
         self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
-        self.categories = [
-            'computer science', 'math', 'chemistry', 'engineering', 'law', 'biology', 'health', 'physics', 'business',
-            'philosophy', 'economics', 'other', 'psychology', 'history'
-        ]
+
+    def load(self, **kwargs):
+        # default load all data
+        kwargs['subset_list'] = ['default']
+        return super().load(**kwargs)
 
     def gen_prompts(self, data_dict: dict, **kwargs) -> Dict[str, list]:
         """
@@ -37,26 +43,32 @@ class MMLUProAdapter(DataAdapter):
         Return a dict with category as key and list of prompts as value.
         """
 
-        data_dict = data_dict[self.subset_list[0]]  # Only one subset for MMLU-Pro
+        data_dict = data_dict['default']  # Only one subset for MMLU-Pro
         fewshot_prompts = self.get_fewshot_examples(data_dict)
 
         #  Use the category as key to group the prompts
         res_dict = defaultdict(list)
         # generate prompts for each test sample
         for entry in data_dict[self.eval_split]:
-            prefix = fewshot_prompts[entry['category']]
+            subset_name = entry['category']
+            if subset_name not in self.subset_list:
+                continue
+            prefix = fewshot_prompts[subset_name]
             query = prefix + 'Q: ' + entry['question'] + '\n' + \
                 self.__form_options(entry['options']) + '\n'
 
-            prompt_d = {'data': [query], 'system_prompt': self.prompt_template, AnswerKeys.RAW_INPUT: entry}
+            full_prompt = self.prompt_template.format(subset_name=subset_name, query=query)
+            prompt_d = {'data': [full_prompt], 'system_prompt': self.system_prompt, AnswerKeys.RAW_INPUT: entry}
 
-            res_dict[entry['category']].append(prompt_d)
+            res_dict[subset_name].append(prompt_d)
         return res_dict
 
     def get_fewshot_examples(self, data_dict: dict):
-        # load 5-shot prompts for each category
-        prompts = {c: '' for c in self.categories}
-        for d in data_dict[self.train_split]:
+        # load few-shot prompts for each category
+        prompts = {c: '' for c in self.subset_list}
+        for index, d in enumerate(data_dict[self.train_split]):
+            if index >= self.few_shot_num:
+                break
             prompts[d['category']] += 'Q:' + ' ' + d['question'] + '\n' + \
                 self.__form_options(d['options']) + '\n' + \
                 d['cot_content'] + '\n\n'

evalscope/benchmarks/race/race_adapter.py

@@ -4,7 +4,7 @@ import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy, exact_match
+from evalscope.metrics import exact_match
 from evalscope.models import MultiChoiceModelAdapter
 from evalscope.utils import ResponseParser
 from evalscope.utils.io_utils import jsonl_to_list
@@ -20,7 +20,7 @@ logger = get_logger()
     dataset_id='modelscope/race',
     model_adapter=MultiChoiceModelAdapter,
     subset_list=['high', 'middle'],
-    metric_list=[AverageAccuracy],
+    metric_list=['AverageAccuracy'],
     few_shot_num=3,
     train_split='train',
     eval_split='test',
@@ -82,7 +82,7 @@ class RACEAdapter(DataAdapter):
 
         full_prompt: str = context.strip() + self._generate_prompt(input_d=input_d, include_answer=False)
 
-        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.prompt_template}
+        return {'data': [full_prompt], 'multi_choices': self.choices, 'system_prompt': self.system_prompt}
 
     def get_gold_answer(self, input_d: dict) -> str:
         # Get the gold choice

evalscope/benchmarks/trivia_qa/trivia_qa_adapter.py

@@ -6,7 +6,6 @@ import os
 from evalscope.benchmarks import Benchmark
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy
 from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils import get_logger
 
@@ -20,7 +19,7 @@ logger = get_logger()
     dataset_id='modelscope/trivia_qa',
     model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
-    metric_list=[AverageAccuracy],
+    metric_list=['AverageAccuracy'],
     few_shot_num=5,
     train_split='dev',
     eval_split='test',

evalscope/benchmarks/truthful_qa/truthful_qa_adapter.py

@@ -9,9 +9,8 @@ from typing import List
 from evalscope.benchmarks import Benchmark
 from evalscope.benchmarks.data_adapter import DataAdapter
 from evalscope.constants import EvalType
-from evalscope.metrics import AverageAccuracy
 from evalscope.models import ContinuationLogitsModelAdapter
-from evalscope.utils import get_logger, normalize_score
+from evalscope.utils import get_logger
 
 # flake8: noqa
 
@@ -25,7 +24,7 @@ logger = get_logger()
     dataset_id='modelscope/truthful_qa',
     model_adapter=ContinuationLogitsModelAdapter,
     subset_list=['multiple_choice'],
-    metric_list=[AverageAccuracy],
+    metric_list=['AverageAccuracy'],
     few_shot_num=0,
     train_split=None,
     eval_split='validation',
@@ -284,8 +283,9 @@ class TruthfulQaAdapter(DataAdapter):
                 logger.error(f'** Unknown review_res: {review_res_d}')
 
         # To get mc2 score
-        return [{
-            'metric_name': self.metric_list[0].name,
-            'score': self.metric_list[0].object(mc2_list),
-            'num': len(mc2_list)
-        }]
+        # return [{
+        #     'metric_name': self.metric_list[0].name,
+        #     'score': self.metric_list[0].object(mc2_list),
+        #     'num': len(mc2_list)
+        # }]
+        return super().compute_metric(mc2_list)

evalscope/cli/start_app.py

@@ -3,7 +3,7 @@ import os
 from argparse import ArgumentParser
 
 from evalscope.cli.base import CLICommand
-from evalscope.report.app import create_app
+from evalscope.report.app import add_argument, create_app
 
 
 def subparser_func(args):
@@ -23,7 +23,8 @@ class StartAppCMD(CLICommand):
         """ define args for create pipeline template command.
         """
         parser = parsers.add_parser(StartAppCMD.name)
+        add_argument(parser)
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
-        create_app()
+        create_app(self.args)

evalscope/collections/evaluator.py

@@ -2,14 +2,15 @@ import json
 import os
 import pandas as pd
 from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from tabulate import tabulate
 from tqdm import tqdm
 from typing import List
 
-from evalscope.benchmarks import Benchmark
+from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.collections.sampler import DatasetEntry
 from evalscope.config import TaskConfig
-from evalscope.constants import DataCollection, DumpMode
+from evalscope.constants import AnswerKeys, DumpMode, EvalType
 from evalscope.evaluator import Evaluator
 from evalscope.models import get_local_model, initialize_model_adapter
 from evalscope.report import ReportGenerator
@@ -29,11 +30,16 @@ class SimpleEvaluator(Evaluator):
             task_cfg=task_cfg,
             outputs=outputs)
 
-    def get_answer(self, input_prompt, subset_name, infer_cfg) -> dict:
-        answer_d: dict = self.model_adapter.predict(inputs=input_prompt, infer_cfg=infer_cfg)
-        answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
-        processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
-        return processed_answer
+    def get_answer(self, samples, infer_cfg) -> List[dict]:
+        input_prompts = [sample.prompt for sample in samples]
+        subset_name = samples[0].subset_name
+        answers_list = []
+        answer_ds: List[dict] = self.model_adapter.predict(inputs=input_prompts, infer_cfg=infer_cfg)
+        for answer_d, input_prompt in zip(answer_ds, input_prompts):
+            answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
+            processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
+            answers_list.append(processed_answer)
+        return answers_list, samples
 
     def get_review(self, answer_d) -> dict:
         review_id, reviewer_spec = self._generate_review_id(answer_d)
@@ -42,38 +48,50 @@ class SimpleEvaluator(Evaluator):
 
     def get_score(self, review_d) -> float:
         metric_score: List[dict] = self.compute_metrics(reviews_list=[review_d])
-        # use the first metric by default
-        score = metric_score[0]['score']
-        return score
+        return metric_score
 
 
 class EvaluatorCollection:
 
-    def __init__(self, task_cfg: TaskConfig, outputs: OutputsStructure):
+    def __init__(self, task_cfg: TaskConfig, data_adapter: DataAdapter, outputs: OutputsStructure):
         self.task_cfg = task_cfg
+        self.data_adapter = data_adapter
         self.outputs = outputs
         self.model = get_local_model(task_cfg)
+
         self.dataset, self.dataset_name = self.load()
-        self.dataset_name_map, self.dataset_id_map = self._parse_dataset()
+        self.dataset_name_map = EvaluatorCollection._init_name_map(self.dataset)
+        self.dataset_id_map = EvaluatorCollection._init_id_map(self.dataset)
         self.evaluators = self._initialize_evaluators()
 
     def load(self) -> tuple[list[DatasetEntry], str]:
-        dataset_path = self.task_cfg.dataset_args[DataCollection.NAME]['local_path']
-        dataset_name = os.path.basename(dataset_path).split('.')[0]
-        raw_dataset = jsonl_to_list(dataset_path)
+        dataset_name = os.path.basename(self.data_adapter.dataset_id).split('.')[0]
+        raw_dataset = self.data_adapter.load()
+        # limit the dataset
+        if self.task_cfg.limit:
+            raw_dataset = raw_dataset[:self.task_cfg.limit]
+        # index dataset
         datasets = []
         for sample in raw_dataset:
+            sample['prompt'].update({'index': sample['index']})
             datasets.append(DatasetEntry(**sample))
+
         return datasets, dataset_name
 
-    def _parse_dataset(self):
+    @staticmethod
+    def _init_name_map(dataset):
         dataset_name_map = defaultdict(lambda: defaultdict(list))
-        dataset_id_map = {}
-        for sample in self.dataset:
+        for sample in dataset:
             dataset_name, subset_name = sample.dataset_name, sample.subset_name
             dataset_name_map[dataset_name][subset_name].append(sample.index)
+        return dataset_name_map
+
+    @staticmethod
+    def _init_id_map(dataset):
+        dataset_id_map = {}
+        for sample in dataset:
             dataset_id_map[sample.index] = sample
-        return dataset_name_map, dataset_id_map
+        return dataset_id_map
 
     def _initialize_evaluators(self):
         evaluators = {}
@@ -93,15 +111,16 @@ class EvaluatorCollection:
                 for subset_name, ids in data_map.items():
                     for _id in ids:
                         row_data: DatasetEntry = self.dataset_id_map[_id]
-                        score = scores[_id]
-                        data.append(
-                            dict(
-                                task_type=row_data.task_type,
-                                categories=tuple(row_data.categories),
-                                dataset_name=dataset_name,
-                                subset_name=subset_name,
-                                tags=row_data.tags,
-                                score=score))
+                        for metric in scores[_id]:
+                            data.append(
+                                dict(
+                                    task_type=row_data.task_type,
+                                    categories=tuple(row_data.categories),
+                                    dataset_name=dataset_name,
+                                    subset_name=subset_name,
+                                    tags=row_data.tags,
+                                    metric=metric['metric_name'],
+                                    score=metric['score']))
             return pd.DataFrame(data)
 
         def aggregate_and_sort(df, group_by_cols):
@@ -117,13 +136,13 @@ class EvaluatorCollection:
         df = get_dataframe(scores)
 
         # multi-level aggregation
-        subset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name', 'subset_name'])
-        dataset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name'])
-        task_report_df = aggregate_and_sort(df, ['task_type'])
+        subset_report_df = aggregate_and_sort(df, ['task_type', 'metric', 'dataset_name', 'subset_name'])
+        dataset_report_df = aggregate_and_sort(df, ['task_type', 'metric', 'dataset_name'])
+        task_report_df = aggregate_and_sort(df, ['task_type', 'metric'])
 
         # explode tags to multiple rows
         df_exploded_tags = df.explode('tags')
-        tag_report_df = aggregate_and_sort(df_exploded_tags, ['tags'])
+        tag_report_df = aggregate_and_sort(df_exploded_tags, ['tags', 'metric'])
 
         # process multi-level categories
         df_categories = df.copy()
@@ -132,7 +151,8 @@ class EvaluatorCollection:
         for level in range(max_depth):
             df_categories[f'category{level}'] = df_categories['categories'].apply(lambda x: x[level]
                                                                                   if len(x) > level else '')
-        category_report_df = aggregate_and_sort(df_categories, [f'category{level}' for level in range(max_depth)])
+        category_report_df = aggregate_and_sort(df_categories,
+                                                [f'category{level}' for level in range(max_depth)] + ['metric'])
 
         # convert to dict format
         report_dict = {
@@ -155,16 +175,60 @@ class EvaluatorCollection:
         with open(report_file_path, 'w', encoding='utf-8') as f:
             json.dump(report.to_dict(), f, ensure_ascii=False, indent=4)
 
+    def _filter_answer(self, pred_file_path):
+        answer_dict = defaultdict(dict)
+        if self.task_cfg.use_cache and os.path.exists(pred_file_path):
+            answers_list = jsonl_to_list(pred_file_path)
+            indices = set()
+            for answer in answers_list:
+                index = answer[AnswerKeys.ORIGIN_PROMPT].get('index')
+                answer_dict[index] = answer
+                indices.add(index)
+            data = []
+            for sample in self.dataset:
+                if sample.index not in indices:
+                    data.append(sample)
+            data_map = self._init_name_map(data)
+
+            return answer_dict, data, data_map
+        return answer_dict, self.dataset, self.dataset_name_map
+
     def get_answers(self):
         pred_file_path = os.path.join(self.outputs.predictions_dir, self.task_cfg.model_id,
                                       f'{self.dataset_name}.jsonl')
         os.makedirs(os.path.dirname(pred_file_path), exist_ok=True)
-        answers = defaultdict(dict)
-        for sample in tqdm(self.dataset, desc='Getting answers'):
-            evaluator = self.evaluators[sample.dataset_name]
-            answer_d = evaluator.get_answer(sample.prompt, sample.subset_name, self.task_cfg.generation_config)
-            answers[sample.index] = answer_d
-            dump_jsonl_data(answer_d, pred_file_path, dump_mode=DumpMode.APPEND)
+
+        answers, dataset, dataset_name_map = self._filter_answer(pred_file_path)
+
+        eval_batch_size = self.task_cfg.eval_batch_size
+        with tqdm(total=len(dataset), desc='Getting answers') as pbar:
+            if self.task_cfg.eval_type == EvalType.SERVICE:
+                with ThreadPoolExecutor(max_workers=eval_batch_size) as executor:
+                    futures = []
+                    for sample in dataset:
+                        evaluator = self.evaluators[sample.dataset_name]
+                        futures.append(executor.submit(evaluator.get_answer, [sample], self.task_cfg.generation_config))
+                    for future in as_completed(futures):
+                        answer_list, samples = future.result()
+                        answers[samples[0].index] = answer_list[0]
+                        dump_jsonl_data(answer_list, pred_file_path, dump_mode=DumpMode.APPEND)
+                        pbar.update(1)
+            else:
+                for dataset_name, data_map in dataset_name_map.items():
+                    # get evaluator for the dataset
+                    evaluator = self.evaluators[dataset_name]
+                    for subset_name, ids in data_map.items():
+                        for i in range(0, len(ids), eval_batch_size):
+                            # get batch samples
+                            batch_ids = ids[i:i + eval_batch_size]
+                            batch_samples = [self.dataset_id_map[_id] for _id in batch_ids]
+                            answer_list, _ = evaluator.get_answer(batch_samples, self.task_cfg.generation_config)
+                            # update answers
+                            for j, _id in enumerate(batch_ids):
+                                answers[_id] = answer_list[j]
+                            dump_jsonl_data(answer_list, pred_file_path, dump_mode=DumpMode.APPEND)
+
+                            pbar.update(len(batch_ids))
         return answers
 
     def get_reviews(self, answers):

evalscope/collections/sampler.py

@@ -44,7 +44,8 @@ class Sampler(ABC):
                         dataset_name=dataset.name,
                         subset_name=subset_name,
                     ))
-        sampled_data = random.choices(all_data, k=count)
+        count = min(count, len(all_data))  # avoid sampling more than the dataset size
+        sampled_data = random.sample(all_data, k=count)
         return sampled_data
 
     def _update_index(self, all_data: List[DatasetEntry]) -> List[dict]:

evalscope/collections/schema.py

@@ -19,8 +19,7 @@ class DatasetInfo:
         benchmark_meta = Benchmark.get(self.name)
 
         data_adapter = benchmark_meta.get_data_adapter(config=self.args)
-        data_dict = data_adapter.load(
-            dataset_name_or_path=benchmark_meta.dataset_id, subset_list=benchmark_meta.subset_list)
+        data_dict = data_adapter.load()
         prompts = data_adapter.gen_prompts(data_dict)
         return prompts
 

evalscope/config.py

@@ -54,6 +54,7 @@ class TaskConfig:
     eval_config: Union[str, Dict, None] = None
     stage: str = EvalStage.ALL
     limit: Optional[int] = None
+    eval_batch_size: int = 1
 
     # Cache and working directory arguments
     mem_cache: bool = False  # Deprecated, will be removed in v1.0.0.