evalscope 0.10.1__py3-none-any.whl → 0.12.0__py3-none-any.whl

evalscope/cli/start_perf.py

@@ -3,8 +3,6 @@ import os
 from argparse import ArgumentParser
 
 from evalscope.cli.base import CLICommand
-from evalscope.perf.arguments import add_argument
-from evalscope.perf.main import run_perf_benchmark
 
 
 def subparser_func(args):
@@ -23,9 +21,13 @@ class PerfBenchCMD(CLICommand):
     def define_args(parsers: ArgumentParser):
         """ define args for create pipeline template command.
         """
+        from evalscope.perf.arguments import add_argument
+
         parser = parsers.add_parser(PerfBenchCMD.name)
         add_argument(parser)
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
+        from evalscope.perf.main import run_perf_benchmark
+
         run_perf_benchmark(self.args)

evalscope/collections/evaluator.py

@@ -2,14 +2,15 @@ import json
 import os
 import pandas as pd
 from collections import defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from tabulate import tabulate
 from tqdm import tqdm
 from typing import List
 
-from evalscope.benchmarks import Benchmark
+from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.collections.sampler import DatasetEntry
 from evalscope.config import TaskConfig
-from evalscope.constants import DataCollection, DumpMode
+from evalscope.constants import AnswerKeys, DumpMode, EvalType
 from evalscope.evaluator import Evaluator
 from evalscope.models import get_local_model, initialize_model_adapter
 from evalscope.report import ReportGenerator
@@ -29,11 +30,16 @@ class SimpleEvaluator(Evaluator):
             task_cfg=task_cfg,
             outputs=outputs)
 
-    def get_answer(self, input_prompt, subset_name, infer_cfg) -> dict:
-        answer_d: dict = self.model_adapter.predict(inputs=input_prompt, infer_cfg=infer_cfg)
-        answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
-        processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
-        return processed_answer
+    def get_answer(self, samples, infer_cfg) -> List[dict]:
+        input_prompts = [sample.prompt for sample in samples]
+        subset_name = samples[0].subset_name
+        answers_list = []
+        answer_ds: List[dict] = self.model_adapter.predict(inputs=input_prompts, infer_cfg=infer_cfg)
+        for answer_d, input_prompt in zip(answer_ds, input_prompts):
+            answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
+            processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
+            answers_list.append(processed_answer)
+        return answers_list, samples
 
     def get_review(self, answer_d) -> dict:
         review_id, reviewer_spec = self._generate_review_id(answer_d)
@@ -42,38 +48,50 @@ class SimpleEvaluator(Evaluator):
 
     def get_score(self, review_d) -> float:
         metric_score: List[dict] = self.compute_metrics(reviews_list=[review_d])
-        # use the first metric by default
-        score = metric_score[0]['score']
-        return score
+        return metric_score
 
 
 class EvaluatorCollection:
 
-    def __init__(self, task_cfg: TaskConfig, outputs: OutputsStructure):
+    def __init__(self, task_cfg: TaskConfig, data_adapter: DataAdapter, outputs: OutputsStructure):
         self.task_cfg = task_cfg
+        self.data_adapter = data_adapter
         self.outputs = outputs
         self.model = get_local_model(task_cfg)
+
         self.dataset, self.dataset_name = self.load()
-        self.dataset_name_map, self.dataset_id_map = self._parse_dataset()
+        self.dataset_name_map = EvaluatorCollection._init_name_map(self.dataset)
+        self.dataset_id_map = EvaluatorCollection._init_id_map(self.dataset)
         self.evaluators = self._initialize_evaluators()
 
     def load(self) -> tuple[list[DatasetEntry], str]:
-        dataset_path = self.task_cfg.dataset_args[DataCollection.NAME]['local_path']
-        dataset_name = os.path.basename(dataset_path).split('.')[0]
-        raw_dataset = jsonl_to_list(dataset_path)
+        dataset_name = os.path.basename(self.data_adapter.dataset_id).split('.')[0]
+        raw_dataset = self.data_adapter.load()
+        # limit the dataset
+        if self.task_cfg.limit:
+            raw_dataset = raw_dataset[:self.task_cfg.limit]
+        # index dataset
         datasets = []
         for sample in raw_dataset:
+            sample['prompt'].update({'index': sample['index']})
             datasets.append(DatasetEntry(**sample))
+
         return datasets, dataset_name
 
-    def _parse_dataset(self):
+    @staticmethod
+    def _init_name_map(dataset):
         dataset_name_map = defaultdict(lambda: defaultdict(list))
-        dataset_id_map = {}
-        for sample in self.dataset:
+        for sample in dataset:
             dataset_name, subset_name = sample.dataset_name, sample.subset_name
             dataset_name_map[dataset_name][subset_name].append(sample.index)
+        return dataset_name_map
+
+    @staticmethod
+    def _init_id_map(dataset):
+        dataset_id_map = {}
+        for sample in dataset:
             dataset_id_map[sample.index] = sample
-        return dataset_name_map, dataset_id_map
+        return dataset_id_map
 
     def _initialize_evaluators(self):
         evaluators = {}
@@ -93,15 +111,16 @@ class EvaluatorCollection:
                 for subset_name, ids in data_map.items():
                     for _id in ids:
                         row_data: DatasetEntry = self.dataset_id_map[_id]
-                        score = scores[_id]
-                        data.append(
-                            dict(
-                                task_type=row_data.task_type,
-                                categories=tuple(row_data.categories),
-                                dataset_name=dataset_name,
-                                subset_name=subset_name,
-                                tags=row_data.tags,
-                                score=score))
+                        for metric in scores[_id]:
+                            data.append(
+                                dict(
+                                    task_type=row_data.task_type,
+                                    categories=tuple(row_data.categories),
+                                    dataset_name=dataset_name,
+                                    subset_name=subset_name,
+                                    tags=row_data.tags,
+                                    metric=metric['metric_name'],
+                                    score=metric['score']))
             return pd.DataFrame(data)
 
         def aggregate_and_sort(df, group_by_cols):
@@ -117,13 +136,13 @@ class EvaluatorCollection:
         df = get_dataframe(scores)
 
         # multi-level aggregation
-        subset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name', 'subset_name'])
-        dataset_report_df = aggregate_and_sort(df, ['task_type', 'dataset_name'])
-        task_report_df = aggregate_and_sort(df, ['task_type'])
+        subset_report_df = aggregate_and_sort(df, ['task_type', 'metric', 'dataset_name', 'subset_name'])
+        dataset_report_df = aggregate_and_sort(df, ['task_type', 'metric', 'dataset_name'])
+        task_report_df = aggregate_and_sort(df, ['task_type', 'metric'])
 
         # explode tags to multiple rows
         df_exploded_tags = df.explode('tags')
-        tag_report_df = aggregate_and_sort(df_exploded_tags, ['tags'])
+        tag_report_df = aggregate_and_sort(df_exploded_tags, ['tags', 'metric'])
 
         # process multi-level categories
         df_categories = df.copy()
@@ -132,7 +151,8 @@ class EvaluatorCollection:
         for level in range(max_depth):
             df_categories[f'category{level}'] = df_categories['categories'].apply(lambda x: x[level]
                                                                                   if len(x) > level else '')
-        category_report_df = aggregate_and_sort(df_categories, [f'category{level}' for level in range(max_depth)])
+        category_report_df = aggregate_and_sort(df_categories,
+                                                [f'category{level}' for level in range(max_depth)] + ['metric'])
 
         # convert to dict format
         report_dict = {
@@ -155,21 +175,71 @@ class EvaluatorCollection:
         with open(report_file_path, 'w', encoding='utf-8') as f:
             json.dump(report.to_dict(), f, ensure_ascii=False, indent=4)
 
+    def _filter_answer(self, pred_file_path):
+        answer_dict = defaultdict(dict)
+        if self.task_cfg.use_cache and os.path.exists(pred_file_path):
+            answers_list = jsonl_to_list(pred_file_path)
+            indices = set()
+            for answer in answers_list:
+                index = answer[AnswerKeys.ORIGIN_PROMPT].get('index')
+                answer_dict[index] = answer
+                indices.add(index)
+            data = []
+            for sample in self.dataset:
+                if sample.index not in indices:
+                    data.append(sample)
+            data_map = self._init_name_map(data)
+
+            return answer_dict, data, data_map
+        return answer_dict, self.dataset, self.dataset_name_map
+
     def get_answers(self):
         pred_file_path = os.path.join(self.outputs.predictions_dir, self.task_cfg.model_id,
                                       f'{self.dataset_name}.jsonl')
         os.makedirs(os.path.dirname(pred_file_path), exist_ok=True)
-        answers = defaultdict(dict)
-        for sample in tqdm(self.dataset, desc='Getting answers'):
-            evaluator = self.evaluators[sample.dataset_name]
-            answer_d = evaluator.get_answer(sample.prompt, sample.subset_name, self.task_cfg.generation_config)
-            answers[sample.index] = answer_d
-            dump_jsonl_data(answer_d, pred_file_path, dump_mode=DumpMode.APPEND)
+
+        answers, dataset, dataset_name_map = self._filter_answer(pred_file_path)
+
+        eval_batch_size = self.task_cfg.eval_batch_size
+        with tqdm(total=len(dataset), desc='Getting answers') as pbar:
+            if self.task_cfg.eval_type == EvalType.SERVICE:
+                with ThreadPoolExecutor(max_workers=eval_batch_size) as executor:
+                    futures = []
+                    for sample in dataset:
+                        evaluator = self.evaluators[sample.dataset_name]
+                        futures.append(executor.submit(evaluator.get_answer, [sample], self.task_cfg.generation_config))
+                    for future in as_completed(futures):
+                        answer_list, samples = future.result()
+                        answers[samples[0].index] = answer_list[0]
+                        dump_jsonl_data(answer_list, pred_file_path, dump_mode=DumpMode.APPEND)
+                        pbar.update(1)
+            else:
+                for dataset_name, data_map in dataset_name_map.items():
+                    # get evaluator for the dataset
+                    evaluator = self.evaluators[dataset_name]
+                    for subset_name, ids in data_map.items():
+                        for i in range(0, len(ids), eval_batch_size):
+                            # get batch samples
+                            batch_ids = ids[i:i + eval_batch_size]
+                            batch_samples = [self.dataset_id_map[_id] for _id in batch_ids]
+                            answer_list, _ = evaluator.get_answer(batch_samples, self.task_cfg.generation_config)
+                            # update answers
+                            for j, _id in enumerate(batch_ids):
+                                answers[_id] = answer_list[j]
+                            dump_jsonl_data(answer_list, pred_file_path, dump_mode=DumpMode.APPEND)
+
+                            pbar.update(len(batch_ids))
         return answers
 
     def get_reviews(self, answers):
         review_file_path = os.path.join(self.outputs.reviews_dir, self.task_cfg.model_id)
         os.makedirs(review_file_path, exist_ok=True)
+
+        if self.task_cfg.use_cache and os.path.exists(review_file_path):
+            logger.warning(
+                f'Ignore use_cache={self.task_cfg.use_cache}, updating the review file: {review_file_path} ...')
+            os.remove(review_file_path)
+
         reviews = defaultdict(dict)
         for sample in tqdm(self.dataset, desc='Getting reviews'):
             evaluator = self.evaluators[sample.dataset_name]

evalscope/collections/sampler.py

@@ -44,7 +44,8 @@ class Sampler(ABC):
                         dataset_name=dataset.name,
                         subset_name=subset_name,
                     ))
-        sampled_data = random.choices(all_data, k=count)
+        count = min(count, len(all_data))  # avoid sampling more than the dataset size
+        sampled_data = random.sample(all_data, k=count)
         return sampled_data
 
     def _update_index(self, all_data: List[DatasetEntry]) -> List[dict]:

evalscope/collections/schema.py

@@ -19,8 +19,7 @@ class DatasetInfo:
         benchmark_meta = Benchmark.get(self.name)
 
         data_adapter = benchmark_meta.get_data_adapter(config=self.args)
-        data_dict = data_adapter.load(
-            dataset_name_or_path=benchmark_meta.dataset_id, subset_list=benchmark_meta.subset_list)
+        data_dict = data_adapter.load()
         prompts = data_adapter.gen_prompts(data_dict)
         return prompts
 

evalscope/config.py

@@ -17,7 +17,7 @@ logger = get_logger()
 
 cur_path = os.path.dirname(os.path.abspath(__file__))
 
-DEFAULT_MODEL_ARGS = {'revision': 'master', 'precision': 'torch.float16', 'device': 'auto'}
+DEFAULT_MODEL_ARGS = {'revision': 'master', 'precision': 'torch.float16'}
 DEFAULT_GENERATION_CONFIG = {
     'max_length': 2048,
     'max_new_tokens': 512,
@@ -54,6 +54,7 @@ class TaskConfig:
     eval_config: Union[str, Dict, None] = None
     stage: str = EvalStage.ALL
     limit: Optional[int] = None
+    eval_batch_size: int = 1
 
     # Cache and working directory arguments
     mem_cache: bool = False  # Deprecated, will be removed in v1.0.0.
@@ -67,6 +68,8 @@ class TaskConfig:
     seed: Optional[int] = 42
     api_url: Optional[str] = None  # Only used for server model
     api_key: Optional[str] = 'EMPTY'  # Only used for server model
+    timeout: Optional[float] = None  # Only used for server model
+    stream: bool = False  # Only used for server model
 
     def __post_init__(self):
         if (not self.model_id) and self.model:

evalscope/evaluator/evaluator.py

@@ -3,15 +3,16 @@
 import json
 import os
 import time
-from collections import OrderedDict
+from collections import OrderedDict, defaultdict
+from concurrent.futures import ThreadPoolExecutor, as_completed
 from copy import deepcopy
 from tqdm import tqdm
 from typing import Any, Dict, List, Optional, Union
 
 from evalscope.benchmarks import DataAdapter
 from evalscope.config import TaskConfig
-from evalscope.constants import AnswerKeys, DumpMode, EvalStage, ReviewKeys
-from evalscope.models import BaseModelAdapter, CustomModelAdapter
+from evalscope.constants import AnswerKeys, DumpMode, EvalStage, EvalType, ReviewKeys
+from evalscope.models import BaseModelAdapter
 from evalscope.report import Report, gen_table
 from evalscope.utils import dict_torch_dtype_to_str, gen_hash
 from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, jsonl_to_list
@@ -36,7 +37,6 @@ class Evaluator(object):
     """
 
     def __init__(self,
-                 dataset_name_or_path: str,
                  data_adapter: DataAdapter,
                  model_adapter: BaseModelAdapter,
                  outputs: OutputsStructure = None,
@@ -44,7 +44,7 @@ class Evaluator(object):
                  **kwargs):
 
         self.dataset_name = data_adapter.name
-        self.dataset_name_or_path = os.path.expanduser(dataset_name_or_path)
+        self.dataset_name_or_path = os.path.expanduser(data_adapter.dataset_id)
         self.model_name = task_cfg.model_id
         self.custom_task_name = f'{self.model_name}_{self.dataset_name}'
 
@@ -63,15 +63,20 @@ class Evaluator(object):
 
     def load_dataset(self):
         dataset = self.data_adapter.load(
-            dataset_name_or_path=self.dataset_name_or_path,
-            subset_list=self.data_adapter.subset_list,
-            work_dir=os.path.expanduser(self.task_cfg.dataset_dir),
-            datasets_hub=self.dataset_hub,
-            **self.kwargs)
+            work_dir=os.path.expanduser(self.task_cfg.dataset_dir), datasets_hub=self.dataset_hub, **self.kwargs)
 
         # Get prompts from dataset
         prompts = self.data_adapter.gen_prompts(data_dict=dataset)
-        return prompts
+
+        # Limit and index prompts
+        limited_prompts = defaultdict(list)
+        for subset_name, prompts_list in prompts.items():
+            limit = self.task_cfg.limit or len(prompts_list)
+            for index, prompt in enumerate(prompts_list[:limit]):
+                prompt['index'] = index
+                limited_prompts[subset_name].append(prompt)
+
+        return limited_prompts
 
     def _generate_answer_id(self, model_cfg, input_d, infer_cfg):
         model_cfg_str = json.dumps(OrderedDict(sorted(dict_torch_dtype_to_str(model_cfg).items())), ensure_ascii=False)
@@ -87,12 +92,38 @@ class Evaluator(object):
         answer_d[AnswerKeys.ORIGIN_PROMPT] = input_d
         return answer_d
 
-    def get_answers(self,
-                    subset_name: str,
-                    prompts_list: List[dict],
-                    infer_cfg: dict = None,
-                    debug: bool = False,
-                    **kwargs) -> list:
+    def _get_answer(self, input_prompts, subset_name, infer_cfg) -> List[dict]:
+        answers_list = []
+        answer_ds: List[dict] = self.model_adapter.predict(inputs=input_prompts, infer_cfg=infer_cfg)
+        for answer_d, input_prompt in zip(answer_ds, input_prompts):
+            answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
+            processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
+            answers_list.append(processed_answer)
+        return answers_list
+
+    @staticmethod
+    def filter_answer(use_cache, prompts_list, pred_file_path) -> dict:
+        # Filter prompts that have been answered
+        answers_list = []
+        if not use_cache or not os.path.exists(pred_file_path):
+            return answers_list, prompts_list
+
+        def get_answered_indices(answers_list: List[Dict]) -> List[int]:
+            indices = [answer[AnswerKeys.ORIGIN_PROMPT].get('index') for answer in answers_list]
+
+            if all(index is None for index in indices):
+                return list(range(len(answers_list)))
+
+            return [index for index in indices if index is not None]
+
+        answers_list = jsonl_to_list(pred_file_path)
+        answered_indices = set(get_answered_indices(answers_list))
+        logger.info(f'Reusing predictions from {pred_file_path}, got {len(answered_indices)} answers.')
+
+        prompts = [prompt for i, prompt in enumerate(prompts_list) if i not in answered_indices]
+        return answers_list, prompts
+
+    def get_answers(self, subset_name: str, prompts_list: List[dict], infer_cfg: dict = None, **kwargs) -> list:
         """
         Get answers from model inference.
         It is required to rewrite this method to support your own evaluator.
@@ -110,7 +141,6 @@ class Evaluator(object):
                     max_length: int, the max length of the sequence to be generated.
                     max_new_tokens: int, the max number of new tokens to be generated.
                     repetition_penalty: float, the parameter for repetition penalty. 1.0 means no penalty.
-            debug: whether to run in debug mode.
             **kwargs: kwargs.
 
         Returns: The list of answers.
@@ -119,41 +149,35 @@ class Evaluator(object):
         assert self.model_adapter is not None, 'model must be provided when calling func get_answers() !'
         assert len(prompts_list) > 0, 'prompts_list must not be empty when calling func get_answers() !'
 
-        answers_list = []
         pred_file_name = self.dataset_name + '_' + subset_name + '.jsonl'
         pred_file_path = os.path.join(self.outputs_structure.predictions_dir, self.model_name, pred_file_name)
         os.makedirs(os.path.dirname(pred_file_path), exist_ok=True)
 
-        if self.use_cache and os.path.exists(pred_file_path):
-            answers_list = jsonl_to_list(pred_file_path)
-            logger.info(f'Reusing predictions from {pred_file_path}, got {len(answers_list)} answers.')
-            # Note: assume prediction in order of prompts_list
-            prompts_list = prompts_list[len(answers_list):]
-
-        if isinstance(self.model_adapter, CustomModelAdapter):
-            # Batch inference for custom model
-
-            resp_answers_list: List[Dict[str, Any]] = self.model_adapter.predict(
-                inputs=prompts_list, infer_cfg=infer_cfg)
-
-            for input_prompt, answer_d in zip(prompts_list, resp_answers_list):
-                answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
-                processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
-                answers_list.append(processed_answer)
-                dump_jsonl_data(processed_answer, pred_file_path, dump_mode=DumpMode.APPEND)
-
+        answers_list, prompts_list = Evaluator.filter_answer(self.use_cache, prompts_list, pred_file_path)
+
+        eval_batch_size = self.task_cfg.eval_batch_size
+        if self.task_cfg.eval_type == EvalType.SERVICE:
+            with tqdm(total=len(prompts_list), desc=f'Predicting({subset_name}): ') as pbar:
+                with ThreadPoolExecutor(max_workers=eval_batch_size) as executor:
+                    futures = []
+                    for input_prompt in prompts_list:
+                        futures.append(executor.submit(self._get_answer, [input_prompt], subset_name, infer_cfg))
+                    for future in as_completed(futures):
+                        answer_ds: List[dict] = future.result()
+                        answers_list.extend(answer_ds)
+                        dump_jsonl_data(answer_ds, pred_file_path, dump_mode=DumpMode.APPEND)
+                        pbar.update(len(answer_ds))
         else:
-            for input_prompt in tqdm(prompts_list, total=len(prompts_list), desc=f'Predicting({subset_name}): '):
-                answer_d: dict = self.model_adapter.predict(inputs=input_prompt, infer_cfg=infer_cfg)
-                answer_id = self._generate_answer_id(self.model_adapter.model_cfg, input_prompt, infer_cfg)
-                processed_answer = self._process_answer(answer_d, input_prompt, subset_name, answer_id)
-
-                if debug:
-                    logger.info(f'**input_prompt: {json.dumps(input_prompt, ensure_ascii=False)} \n')
-                    logger.info(f'**predicted ans: {json.dumps(processed_answer, ensure_ascii=False)} \n')
-
-                answers_list.append(processed_answer)
-                dump_jsonl_data(processed_answer, pred_file_path, dump_mode=DumpMode.APPEND)
+            batch_prompts_list = [
+                prompts_list[i:i + eval_batch_size] for i in range(0, len(prompts_list), eval_batch_size)
+            ]
+            with tqdm(total=len(prompts_list), desc=f'Predicting({subset_name}): ') as pbar:
+                for batch_prompts in batch_prompts_list:
+                    answer_ds: List[dict] = self._get_answer(
+                        input_prompts=batch_prompts, subset_name=subset_name, infer_cfg=infer_cfg)
+                    answers_list.extend(answer_ds)
+                    dump_jsonl_data(answer_ds, pred_file_path, dump_mode=DumpMode.APPEND)
+                    pbar.update(len(batch_prompts))
 
         logger.info(f'Dump predictions to {pred_file_path}.')
         return answers_list
@@ -200,17 +224,13 @@ class Evaluator(object):
     def _generate_review_id(self, answer_d):
         # Gen review_id (concat: answer_id + reviewer_spec)
         answer_id = answer_d[AnswerKeys.ANSWER_ID]
-        reviewer_spec = {
-            'metric': [metric.name for metric in self.data_adapter.metric_list],
-            'reviewer': ['Evaluator'],
-            'revision': ['default']
-        }
+        reviewer_spec = {'metric': self.data_adapter.metric_list, 'reviewer': ['Evaluator'], 'revision': ['default']}
         reviewer_spec_str = json.dumps(
             OrderedDict(sorted(dict_torch_dtype_to_str(reviewer_spec).items())), ensure_ascii=False)
         review_id = 'review-' + gen_hash(answer_id + reviewer_spec_str)
         return review_id, reviewer_spec
 
-    def get_reviews(self, subset_name: str, answers_list: List[dict], debug: bool = False, **kwargs) -> list:
+    def get_reviews(self, subset_name: str, answers_list: List[dict], **kwargs) -> list:
         """
         Get reviews from answers.
         It is required to rewrite this method to support your own evaluator.
@@ -218,7 +238,6 @@ class Evaluator(object):
         Args:
             subset_name: subset name of benchmark
             answers_list: inference results list.
-            debug: whether to run in debug mode.
             **kwargs: kwargs.
 
         Returns: reviews list.
@@ -231,14 +250,14 @@ class Evaluator(object):
 
         if self.use_cache and os.path.exists(review_file_path):
             logger.warning(f'Ignore use_cache={self.use_cache}, updating the review file: {review_file_path} ...')
+            os.remove(review_file_path)
 
         for answer_d in tqdm(answers_list, total=len(answers_list), desc=f'Reviewing({subset_name}): '):
             review_id, reviewer_spec = self._generate_review_id(answer_d)
             # Get review
             review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
 
-            if debug:
-                logger.info(review_d)
+            logger.debug(review_d)
 
             reviews_list.append(review_d)
             # Dump reviews
@@ -274,7 +293,8 @@ class Evaluator(object):
 
             review_res_list.append(review_res)
 
-        metric_score: List[dict] = self.data_adapter.compute_metric(review_res_list=review_res_list)
+        metric_score: List[dict] = self.data_adapter.compute_metric(
+            review_res_list=review_res_list, reviews_list=reviews_list)
 
         return metric_score
 
@@ -315,7 +335,7 @@ class Evaluator(object):
                 logger.error('Failed to generate report table.')
         return report_map
 
-    def eval(self, infer_cfg: dict = None, debug: bool = False, **kwargs) -> dict:
+    def eval(self, **kwargs) -> dict:
         """
         Evaluate the model on the specific benchmark. Streaming & parallel mode is supported.
         It is required to rewrite this method to support your own evaluator.
@@ -329,7 +349,6 @@ class Evaluator(object):
 
         Args:
             infer_cfg: The config for model inference.
-            debug: Whether to run in debug mode. Default: False.
 
         Returns:
             Dict of results. Depends on the stage of evaluation.
@@ -347,17 +366,14 @@ class Evaluator(object):
 
         prompts = self.load_dataset()
         for subset_name, prompts_list in prompts.items():
-            limit = kwargs.get('limit', len(prompts_list))
-            prompts_list = prompts_list[:limit]
 
             answers_list: list = self.get_answers(
-                subset_name=subset_name, prompts_list=prompts_list, infer_cfg=infer_cfg, debug=debug, **kwargs)
+                subset_name=subset_name, prompts_list=prompts_list, infer_cfg=self.task_cfg.generation_config, **kwargs)
             if self.stage == EvalStage.INFER:
                 stage_answers_dict[subset_name] = answers_list
                 continue
 
-            reviews_list: list = self.get_reviews(
-                subset_name=subset_name, answers_list=answers_list, debug=debug, **kwargs)
+            reviews_list: list = self.get_reviews(subset_name=subset_name, answers_list=answers_list, **kwargs)
 
             metric_res = self.compute_metrics(reviews_list=reviews_list)
             reviews_score_all[subset_name] = metric_res

evalscope/metrics/__init__.py

@@ -1,4 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-from evalscope.metrics.metrics import bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean, weighted_mean
+from evalscope.metrics.metrics import (bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean,
+                                       simple_f1_score, weighted_mean)
 from evalscope.metrics.named_metrics import *
 from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh