evalscope 0.11.0__py3-none-any.whl → 0.12.0__py3-none-any.whl

evalscope/arguments.py

@@ -71,6 +71,8 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--seed', type=int, default=42, help='Random seed for reproducibility.')
     parser.add_argument('--api-key', type=str, default='EMPTY', help='The API key for the remote API model.')
     parser.add_argument('--api-url', type=str, default=None, help='The API url for the remote API model.')
+    parser.add_argument('--timeout', type=float, default=None, help='The timeout for the remote API model.')
+    parser.add_argument('--stream', action='store_true', default=False, help='Stream mode.')  # noqa: E501
     # yapf: enable
 
 

evalscope/benchmarks/aime/aime25_adapter.py

@@ -0,0 +1,49 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
+from evalscope.models import ChatGenerationModelAdapter
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='aime25',
+    dataset_id='TIGER-Lab/AIME25',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['default'],
+    metric_list=['AveragePass@1'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='train',  # Only train set is available
+    prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+)
+class AIME25Adapter(DataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate the prompt for the model input.
+        """
+        problem = input_d['question']
+        full_prompt = self.prompt_template.format(query=problem)
+
+        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Extract the gold answer from the input dict.
+        return strip_answer_string(input_d['answer'])
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+        """
+        Parse the model output to get the answer. Could be the best choice index.
+        """
+        # Note: Use same extraction method for both of checkpoint/service/custom
+        result = strip_answer_string(extract_answer(result))
+        return result
+
+    def match(self, gold: str, pred: str) -> float:
+        return math_equal(pred, gold)

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -171,11 +171,6 @@ class BBHAdapter(DataAdapter):
                 prompt_d[AnswerKeys.RAW_INPUT] = sample_d_new
                 res_dict[sub_name].append(prompt_d)
 
-        rnd = random.Random()
-        rnd.seed(42)
-        for k, v in res_dict.items():
-            rnd.shuffle(v)
-
         return res_dict
 
     def get_gold_answer(self, input_d: dict) -> str:

evalscope/benchmarks/benchmark.py

@@ -24,6 +24,8 @@ class BenchmarkMeta:
     eval_split: Optional[str] = None
     prompt_template: Optional[str] = None
     system_prompt: Optional[str] = None
+    query_template: Optional[str] = None
+    pretty_name: Optional[str] = None
 
     def _update(self, args: dict):
         if args.get('local_path'):
@@ -59,7 +61,7 @@ class Benchmark:
     @classmethod
     def get(cls, name: str) -> 'BenchmarkMeta':
         if name not in BENCHMARK_MAPPINGS:
-            raise Exception(f'Unknown benchmark: {name}. Available tasks: {BENCHMARK_MAPPINGS.keys()}')
+            raise Exception(f'Unknown benchmark: {name}. Available tasks: {list(BENCHMARK_MAPPINGS.keys())}')
         benchmark = BENCHMARK_MAPPINGS[name]
         return benchmark
 

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -23,7 +23,7 @@ logger = get_logger()
     subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
     metric_list=['AveragePass@1'],
     few_shot_num=4,
-    train_split='train',
+    train_split=None,
     eval_split='test',
     prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
 )
@@ -43,7 +43,8 @@ class CompetitionMathAdapter(DataAdapter):
     def load(self, **kwargs):
         # default load all levels
         kwargs['subset_list'] = ['default']
-        return super().load(**kwargs)
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='level')
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         data_dict = defaultdict(dict)
@@ -63,21 +64,6 @@ class CompetitionMathAdapter(DataAdapter):
 
         return data_dict
 
-    def gen_prompts(self, data_dict: dict) -> dict:
-        res_dict: dict = defaultdict(list)
-
-        #  use level as subset
-        for sub_name, sub_data_dict in data_dict.items():
-            for sample_d in sub_data_dict[self.eval_split]:
-                level = sample_d['level']
-                if level not in self.subset_list:
-                    continue
-                prompt_d = self.gen_prompt(input_d=sample_d, few_shot_list=None)
-                prompt_d[AnswerKeys.RAW_INPUT] = sample_d
-                res_dict[level].append(prompt_d)
-
-        return res_dict
-
     def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
         """
         Generate the prompt for the model input.

evalscope/benchmarks/data_adapter.py

@@ -2,6 +2,7 @@
 import os.path
 import random
 from abc import ABC, abstractmethod
+from collections import defaultdict
 from typing import Any, List, Optional, Union
 
 from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, EvalType, HubType
@@ -24,6 +25,8 @@ class DataAdapter(ABC):
                  eval_split: Optional[str] = None,
                  prompt_template: Optional[str] = None,
                  system_prompt: Optional[str] = None,
+                 query_template: Optional[str] = None,
+                 pretty_name: Optional[str] = None,
                  **kwargs):
         """
         Data Adapter for the benchmark. You need to implement the following methods:
@@ -52,6 +55,8 @@ class DataAdapter(ABC):
         self.eval_split = eval_split
         self.prompt_template = prompt_template
         self.system_prompt = system_prompt
+        self.query_template = query_template
+        self.pretty_name = pretty_name
         self.config_kwargs = kwargs
         self.category_map = kwargs.get('category_map', {})
 
@@ -59,7 +64,6 @@ class DataAdapter(ABC):
              dataset_name_or_path: str = None,
              subset_list: list = None,
              work_dir: Optional[str] = DEFAULT_DATASET_CACHE_DIR,
-             datasets_hub: str = HubType.MODELSCOPE,
              **kwargs) -> dict:
         """
         Load the dataset. Remote and local datasets are supported.
@@ -74,22 +78,40 @@ class DataAdapter(ABC):
 
         # Try to load dataset from local disk
         if os.path.exists(dataset_name_or_path):
-            logger.info(f'Loading dataset from work_dir: {work_dir}: > dataset_name: {dataset_name_or_path} > \
-                    subsets: {subset_list}')
             data_dict = self.load_from_disk(dataset_name_or_path, subset_list, work_dir, **kwargs)
-            if len(data_dict) == 0 or len(next(iter(data_dict.values()))) == 0:
-                raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
         else:
-            from modelscope.msdatasets import MsDataset
+            data_dict = self.load_from_hub(dataset_name_or_path, subset_list, work_dir, **kwargs)
+        if len(data_dict) == 0 or len(next(iter(data_dict.values()))) == 0:
+            raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
+        return data_dict
+
+    def load_from_hub(self, dataset_name_or_path: str, subset_list: list, work_dir: str, **kwargs) -> dict:
+        from modelscope.msdatasets import MsDataset
+
+        datasets_hub: str = kwargs.pop('datasets_hub', HubType.MODELSCOPE)
+        split_as_subset: bool = kwargs.pop('split_as_subset', False)
+        # Load dataset from remote
+        logger.info(
+            f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
 
-            # Load dataset from remote
-            logger.info(
-                f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path} > subsets: {subset_list}')
-            data_dict = {}
-            split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
-            if len(split_list) == 0:
-                logger.error(f'Got empty split list: {split_list}')
+        data_dict = {}
+        split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
+        if len(split_list) == 0:
+            logger.error(f'Got empty split list: {split_list}')
 
+        if split_as_subset:
+            for sub_name in subset_list:
+                data_dict[sub_name] = {}
+                # e.g. train: few-shot, test: target dataset to evaluate
+                for split in split_list:
+                    dataset = MsDataset.load(
+                        dataset_name=dataset_name_or_path,
+                        split=sub_name,  # load subset from split
+                        cache_dir=work_dir,
+                        hub=datasets_hub,
+                        **kwargs)
+                    data_dict[sub_name].update({split: dataset})
+        else:
             for sub_name in subset_list:
                 data_dict[sub_name] = {}
                 # e.g. train: few-shot, test: target dataset to evaluate
@@ -101,17 +123,48 @@ class DataAdapter(ABC):
                         cache_dir=work_dir,
                         hub=datasets_hub,
                         **kwargs)
-
                     data_dict[sub_name].update({split: dataset})
 
         return data_dict
 
-    def load_from_disk(self, *args, **kwargs) -> dict:
+    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
         """
         Load the dataset from local disk.
         If you want to support local dataset, please rewrite this method in xxx_data_adapter.
+        Use modelscope.msdatasets.MsDataset.load to load the dataset from local by default.
+        """
+        from modelscope.msdatasets import MsDataset
+
+        logger.info(f'Loading dataset from work_dir: {work_dir}: > dataset_name: {dataset_name_or_path} > \
+                subsets: {subset_list}')
+        data_dict = {}
+        subset_list = subset_list or self.subset_list
+        split_list = [split for split in [self.train_split, self.eval_split] if split is not None]
+        for sub_name in subset_list:
+            data_dict[sub_name] = {}
+            # e.g. train: few-shot, test: target dataset to evaluate
+            for split in split_list:
+                dataset = MsDataset.load(
+                    dataset_name=dataset_name_or_path, subset_name=sub_name, split=split, cache_dir=work_dir, **kwargs)
+                data_dict[sub_name].update({split: dataset})
+        return data_dict
+
+    def reformat_subset(self, data_dict: dict, subset_key: str, format: str = '{}') -> dict:
+        """
+        Reformat the dataset subset with subset_key and format.
         """
-        return {}
+        res_dict: dict = defaultdict(lambda: defaultdict(list), {key: defaultdict(list) for key in self.subset_list})
+
+        for sub_name, sub_data_dict in data_dict.items():
+            for split in [self.train_split, self.eval_split]:
+                if split is None:
+                    continue
+                for sample_d in sub_data_dict[split]:
+                    new_subset_name = format.format(sample_d[subset_key])
+                    if new_subset_name not in self.subset_list:
+                        continue
+                    res_dict[new_subset_name][split].append(sample_d)
+        return res_dict
 
     def gen_prompts(self, data_dict: dict) -> dict:
         """
@@ -138,7 +191,7 @@ class DataAdapter(ABC):
 
         for sub_name, sub_data_dict in data_dict.items():
             few_shot_data = []
-            if self.few_shot_num and self.few_shot_num > 0:
+            if self.train_split and self.few_shot_num and self.few_shot_num > 0:
                 few_shot_random: bool = self.config_kwargs.get('few_shot_random', True)
                 few_shot_data = self.get_fewshot_examples([item for item in sub_data_dict[self.train_split]],
                                                           self.few_shot_num,
@@ -161,7 +214,7 @@ class DataAdapter(ABC):
         else:
             return data_list[:k]
 
-    def compute_metric(self, review_res_list: Union[dict, list]) -> List[dict]:
+    def compute_metric(self, review_res_list: Union[dict, list], **kwargs) -> List[dict]:
         """
         Compute evaluation result by specific metrics.
 

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -24,7 +24,7 @@ logger = get_logger()
     train_split='dev',
     eval_split='val',
     prompt_template='请回答问题，并选出其中的正确答案\n{query}',
-)
+    query_template='问题：{question}\n{choices}\n答案: {answer}\n\n')
 class GeneralMCQAdapter(DataAdapter):
 
     choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
@@ -115,15 +115,11 @@ class GeneralMCQAdapter(DataAdapter):
     def match(self, gold: str, pred: str) -> float:
         return exact_match(gold=gold, pred=pred)
 
-    @classmethod
-    def _format_example(cls, input_d: dict, include_answer=True):
-        example = '问题：' + input_d['question']
-        for choice in cls.choices:
-            if choice in input_d:
-                example += f'\n{choice}. {input_d[f"{choice}"]}'
+    def _format_example(self, input_d: dict, include_answer=True):
+        choices_str = '\n'.join([f'{choice}. {input_d[choice]}' for choice in self.choices if choice in input_d])
 
         if include_answer:
-            example += '\n答案: ' + input_d['answer'] + '\n\n'
+            return self.query_template.format(
+                question=input_d['question'], choices=choices_str, answer=input_d['answer'])
         else:
-            example += '\n答案: '
-        return example
+            return self.query_template.format(question=input_d['question'], choices=choices_str, answer='').rstrip()

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -22,6 +22,7 @@ logger = get_logger()
     few_shot_num=0,
     train_split=None,
     eval_split='test',
+    prompt_template='请回答问题\n{query}',
 )
 class GeneralQAAdapter(DataAdapter):
     # TODO: set few_shot_num
@@ -62,10 +63,8 @@ class GeneralQAAdapter(DataAdapter):
             logger.warning('The history is not included in the prompt for GeneralQA. \
                            To be supported in the future.')
 
-        prompt = input_d.get('question', '') or input_d.get('query', '')
-
-        # if len(history) > 0:
-        #     prompt = '\n'.join(history) + '\n' + prompt
+        query = input_d.get('question', '') or input_d.get('query', '')
+        prompt = self.prompt_template.format(query=query)
         return {'data': [prompt], 'system_prompt': self.system_prompt}
 
     def get_gold_answer(self, input_d: dict) -> str:
@@ -107,7 +106,7 @@ class GeneralQAAdapter(DataAdapter):
         res.update(bleu_dict)
         return res
 
-    def compute_metric(self, review_res_list: List[dict]) -> List[dict]:
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
         """
         compute weighted mean of the bleu score of all samples
 

evalscope/benchmarks/gpqa/gpqa_adapter.py

@@ -15,7 +15,7 @@ from evalscope.models import ChatGenerationModelAdapter
     subset_list=['gpqa_extended', 'gpqa_main', 'gpqa_diamond'],
     metric_list=['AveragePass@1'],
     few_shot_num=5,
-    train_split='train',
+    train_split=None,
     eval_split='train',  # only have train split
     prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
 )

evalscope/benchmarks/gsm8k/gsm8k_adapter.py

@@ -20,7 +20,7 @@ logger = get_logger()
     subset_list=['main'],
     metric_list=['AverageAccuracy'],
     few_shot_num=4,
-    train_split='train',
+    train_split=None,
     eval_split='test',
     prompt_template="Question: {query}\nLet's think step by step\nAnswer:",
 )

evalscope/benchmarks/ifeval/ifeval_adapter.py

@@ -47,7 +47,7 @@ class IFEvalAdapter(DataAdapter):
     def match(self, gold: Any, pred: Any) -> Dict:
         return process_results(gold, [pred])
 
-    def compute_metric(self, review_res_list: List[dict]) -> Any:
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> Any:
         # aggregate review results
         res_dict = defaultdict(list)
         for res in review_res_list:

evalscope/benchmarks/math_500/math_500_adapter.py

@@ -1,4 +1,7 @@
+from collections import defaultdict
+
 from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import AnswerKeys
 from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
 from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils.logger import get_logger
@@ -12,7 +15,7 @@ logger = get_logger()
     name='math_500',
     dataset_id='AI-ModelScope/MATH-500',
     model_adapter=ChatGenerationModelAdapter,
-    subset_list=['default'],
+    subset_list=['Level 1', 'Level 2', 'Level 3', 'Level 4', 'Level 5'],
     metric_list=['AveragePass@1'],
     few_shot_num=0,
     train_split=None,
@@ -24,6 +27,12 @@ class Math500Adapter(DataAdapter):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
 
+    def load(self, **kwargs):
+        # default load all levels
+        kwargs['subset_list'] = ['default']
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='level', format='Level {}')
+
     def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
         """
         Generate the prompt for the model input.

evalscope/benchmarks/mmlu_pro/mmlu_pro_adapter.py

@@ -15,7 +15,7 @@ SUBSET_LIST = [
 
 @Benchmark.register(
     name='mmlu_pro',
-    dataset_id='modelscope/mmlu-pro',
+    dataset_id='modelscope/MMLU-Pro',
     model_adapter=ChatGenerationModelAdapter,
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],
@@ -35,41 +35,25 @@ class MMLUProAdapter(DataAdapter):
     def load(self, **kwargs):
         # default load all data
         kwargs['subset_list'] = ['default']
-        return super().load(**kwargs)
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='category')
 
-    def gen_prompts(self, data_dict: dict, **kwargs) -> Dict[str, list]:
-        """
-        Generate model prompt from raw input, unify the prompt format for MMLU-Pro benchmark.
-        Return a dict with category as key and list of prompts as value.
-        """
-
-        data_dict = data_dict['default']  # Only one subset for MMLU-Pro
-        fewshot_prompts = self.get_fewshot_examples(data_dict)
-
-        #  Use the category as key to group the prompts
-        res_dict = defaultdict(list)
-        # generate prompts for each test sample
-        for entry in data_dict[self.eval_split]:
-            subset_name = entry['category']
-            if subset_name not in self.subset_list:
-                continue
-            prefix = fewshot_prompts[subset_name]
-            query = prefix + 'Q: ' + entry['question'] + '\n' + \
-                self.__form_options(entry['options']) + '\n'
-
-            full_prompt = self.prompt_template.format(subset_name=subset_name, query=query)
-            prompt_d = {'data': [full_prompt], 'system_prompt': self.system_prompt, AnswerKeys.RAW_INPUT: entry}
+    def gen_prompt(self, input_d: Dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
+        if self.few_shot_num > 0:
+            prefix = self.format_fewshot_examples(few_shot_list)
+        else:
+            prefix = ''
+        query = prefix + 'Q: ' + input_d['question'] + '\n' + \
+            self.__form_options(input_d['options']) + '\n'
 
-            res_dict[subset_name].append(prompt_d)
-        return res_dict
+        full_prompt = self.prompt_template.format(subset_name=subset_name, query=query)
+        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
 
-    def get_fewshot_examples(self, data_dict: dict):
+    def format_fewshot_examples(self, few_shot_list):
         # load few-shot prompts for each category
-        prompts = {c: '' for c in self.subset_list}
-        for index, d in enumerate(data_dict[self.train_split]):
-            if index >= self.few_shot_num:
-                break
-            prompts[d['category']] += 'Q:' + ' ' + d['question'] + '\n' + \
+        prompts = ''
+        for index, d in enumerate(few_shot_list):
+            prompts += 'Q: ' + d['question'] + '\n' + \
                 self.__form_options(d['options']) + '\n' + \
                 d['cot_content'] + '\n\n'
         return prompts

evalscope/benchmarks/musr/musr_adapter.py

@@ -0,0 +1,68 @@
+import ast
+from typing import Any
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType
+from evalscope.metrics import exact_match
+from evalscope.models import ChatGenerationModelAdapter
+from evalscope.utils.utils import ResponseParser
+
+
+@Benchmark.register(
+    name='musr',
+    pretty_name='MuSR',
+    dataset_id='AI-ModelScope/MuSR',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['murder_mysteries', 'object_placements', 'team_allocation'],
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template=
+    '{narrative}\n\n{question}\n\n{choices}\nThink step by step and then finish your answer with \"the answer is (X)\" where X is the correct letter choice.',  # noqa: E501
+)
+class MuSRAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.choices = ['A', 'B', 'C', 'D', 'E', 'F']
+
+    def load(self, **kwargs):
+        # default load all levels
+        kwargs['split_as_subset'] = True
+        data_dict = super().load(**kwargs)
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
+
+        choices = self.format_choice(ast.literal_eval(input_d['choices']))
+
+        full_prompt = self.prompt_template.format(
+            narrative=input_d['narrative'], question=input_d['question'], choices=choices)
+
+        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+
+    def format_choice(self, options: list):
+        option_str = ''
+        for opt, choice in zip(options, self.choices):
+            option_str += f'({choice}): {opt}\n'
+        return option_str
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return self.choices[input_d['answer_index']]
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        return ResponseParser.parse_first_option(result)
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/process_bench/critique_template.txt

@@ -0,0 +1,13 @@
+The following is a math problem and a solution (split into paragraphs, enclosed with tags and indexed from 0):
+
+[Math Problem]
+
+{problem}
+
+[Solution]
+
+{tagged_response}
+
+Your task is to review and critique the solution paragraph by paragraph. Once you identify an error in a paragraph, return the index of the paragraph where the earliest error occurs. Otherwise, return the index of -1 (which typically denotes "not found").
+
+Please put your final answer (i.e., the index) in \boxed{{}}.

evalscope/benchmarks/process_bench/process_bench_adapter.py

@@ -0,0 +1,96 @@
+import os
+import re
+from typing import Any, List
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import AnswerKeys, EvalType
+from evalscope.metrics import Metric, mean, metric_registry, simple_f1_score
+from evalscope.models import ChatGenerationModelAdapter
+
+cur_path = os.path.dirname(os.path.abspath(__file__))
+
+
+@Benchmark.register(
+    name='process_bench',
+    pretty_name='ProcessBench',
+    dataset_id='Qwen/ProcessBench',
+    model_adapter=ChatGenerationModelAdapter,
+    subset_list=['gsm8k', 'math', 'olympiadbench', 'omnimath'],
+    metric_list=['error_acc', 'correct_acc', 'simple_f1_score'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class ProcessBenchAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.prompt_template = open(os.path.join(cur_path, 'critique_template.txt')).read()
+
+        # register metrics
+        metric_registry.register(Metric(name='error_acc', object=mean))
+        metric_registry.register(Metric(name='correct_acc', object=mean))
+        metric_registry.register(Metric(name='simple_f1_score', object=simple_f1_score))
+
+    def load(self, **kwargs):
+        # default load all levels
+        kwargs['split_as_subset'] = True
+        data_dict = super().load(**kwargs)
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
+
+        problem = input_d['problem']
+        steps = input_d['steps']
+        tagged_response = ''
+        for sdx, step in enumerate(steps):
+            tagged_response += f'<paragraph_{sdx}>\n{step}\n</paragraph_{sdx}>\n\n'
+        tagged_response = tagged_response.strip()
+
+        full_prompt = self.prompt_template.format(problem=problem, tagged_response=tagged_response)
+
+        return {'data': [full_prompt], 'system_prompt': self.system_prompt}
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return int(input_d['label'])
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        pred = ProcessBenchAdapter.extract_answer(result)
+        try:
+            pred = int(pred)
+        except Exception:
+            pred = None
+        return pred
+
+    def match(self, gold: int, pred: int) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        return gold == pred
+
+    def compute_metric(self, review_res_list: list, **kwargs) -> List[dict]:
+        reviews_list = kwargs['reviews_list']
+        error_data = []
+        correct_data = []
+        for res, raw in zip(review_res_list, reviews_list):
+            if raw[AnswerKeys.RAW_INPUT]['label'] == -1:
+                correct_data.append(res)
+            else:
+                error_data.append(res)
+        data = {'error_acc': error_data, 'correct_acc': correct_data, 'simple_f1_score': (correct_data, error_data)}
+        return super().compute_metric(data)
+
+    @staticmethod
+    def extract_answer(solution_text: str):
+        boxed_pattern = r'\\boxed\{([^}]*)\}'
+        matches = re.findall(boxed_pattern, solution_text)
+        if matches:
+            return matches[-1].strip()
+        return None