evalscope 0.11.0__py3-none-any.whl → 0.12.1__py3-none-any.whl

evalscope/benchmarks/super_gpqa/super_gpqa_adapter.py

@@ -0,0 +1,191 @@
+import os
+import random
+import re
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType, OutputType
+from evalscope.metrics import exact_match
+from evalscope.utils import logger
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+
+SUBSET_LIST = [
+    'Electronic Science and Technology', 'Philosophy', 'Traditional Chinese Medicine', 'Applied Economics',
+    'Mathematics', 'Physics', 'Clinical Medicine', 'Computer Science and Technology',
+    'Information and Communication Engineering', 'Control Science and Engineering', 'Theoretical Economics', 'Law',
+    'History', 'Basic Medicine', 'Education', 'Materials Science and Engineering', 'Electrical Engineering',
+    'Systems Science', 'Power Engineering and Engineering Thermophysics', 'Military Science', 'Biology',
+    'Business Administration', 'Language and Literature', 'Public Health and Preventive Medicine', 'Political Science',
+    'Chemistry', 'Hydraulic Engineering', 'Chemical Engineering and Technology', 'Pharmacy', 'Geography', 'Art Studies',
+    'Architecture', 'Forestry Engineering', 'Public Administration', 'Oceanography', 'Journalism and Communication',
+    'Nuclear Science and Technology', 'Weapon Science and Technology', 'Naval Architecture and Ocean Engineering',
+    'Environmental Science and Engineering', 'Transportation Engineering', 'Geology', 'Physical Oceanography',
+    'Musicology', 'Stomatology', 'Aquaculture', 'Mechanical Engineering',
+    'Aeronautical and Astronautical Science and Technology', 'Civil Engineering', 'Mechanics',
+    'Petroleum and Natural Gas Engineering', 'Sociology', 'Food Science and Engineering', 'Agricultural Engineering',
+    'Surveying and Mapping Science and Technology', 'Metallurgical Engineering',
+    'Library, Information and Archival Management', 'Mining Engineering', 'Astronomy',
+    'Geological Resources and Geological Engineering', 'Atmospheric Science', 'Optical Engineering', 'Animal Husbandry',
+    'Geophysics', 'Crop Science', 'Management Science and Engineering', 'Psychology', 'Forestry',
+    'Textile Science and Engineering', 'Veterinary Medicine', 'Instrument Science and Technology', 'Physical Education'
+]
+
+SUBSET_MAPPING = {
+    'Electronic Science and Technology': ['Engineering'],
+    'Philosophy': ['Philosophy'],
+    'Traditional Chinese Medicine': ['Medicine'],
+    'Applied Economics': ['Economics'],
+    'Mathematics': ['Science'],
+    'Physics': ['Science'],
+    'Clinical Medicine': ['Medicine'],
+    'Computer Science and Technology': ['Engineering'],
+    'Information and Communication Engineering': ['Engineering'],
+    'Control Science and Engineering': ['Engineering'],
+    'Theoretical Economics': ['Economics'],
+    'Law': ['Law'],
+    'History': ['History'],
+    'Basic Medicine': ['Medicine'],
+    'Education': ['Education'],
+    'Materials Science and Engineering': ['Engineering'],
+    'Electrical Engineering': ['Engineering'],
+    'Systems Science': ['Science'],
+    'Power Engineering and Engineering Thermophysics': ['Engineering'],
+    'Military Science': ['Military Science'],
+    'Biology': ['Science'],
+    'Business Administration': ['Management'],
+    'Language and Literature': ['Literature and Arts'],
+    'Public Health and Preventive Medicine': ['Medicine'],
+    'Political Science': ['Law'],
+    'Chemistry': ['Science'],
+    'Hydraulic Engineering': ['Engineering'],
+    'Chemical Engineering and Technology': ['Engineering'],
+    'Pharmacy': ['Medicine'],
+    'Geography': ['Science'],
+    'Art Studies': ['Literature and Arts'],
+    'Architecture': ['Engineering'],
+    'Forestry Engineering': ['Engineering'],
+    'Public Administration': ['Management'],
+    'Oceanography': ['Science'],
+    'Journalism and Communication': ['Literature and Arts'],
+    'Nuclear Science and Technology': ['Engineering'],
+    'Weapon Science and Technology': ['Engineering'],
+    'Naval Architecture and Ocean Engineering': ['Engineering'],
+    'Environmental Science and Engineering': ['Engineering'],
+    'Transportation Engineering': ['Engineering'],
+    'Geology': ['Science'],
+    'Physical Oceanography': ['Science'],
+    'Musicology': ['Literature and Arts'],
+    'Stomatology': ['Medicine'],
+    'Aquaculture': ['Agronomy'],
+    'Mechanical Engineering': ['Engineering'],
+    'Aeronautical and Astronautical Science and Technology': ['Engineering'],
+    'Civil Engineering': ['Engineering'],
+    'Mechanics': ['Engineering'],
+    'Petroleum and Natural Gas Engineering': ['Engineering'],
+    'Sociology': ['Sociology'],
+    'Food Science and Engineering': ['Engineering'],
+    'Agricultural Engineering': ['Engineering'],
+    'Surveying and Mapping Science and Technology': ['Engineering'],
+    'Metallurgical Engineering': ['Engineering'],
+    'Library, Information and Archival Management': ['Management'],
+    'Mining Engineering': ['Engineering'],
+    'Astronomy': ['Science'],
+    'Geological Resources and Geological Engineering': ['Engineering'],
+    'Atmospheric Science': ['Science'],
+    'Optical Engineering': ['Engineering'],
+    'Animal Husbandry': ['Agronomy'],
+    'Geophysics': ['Science'],
+    'Crop Science': ['Agronomy'],
+    'Management Science and Engineering': ['Management'],
+    'Psychology': ['Education'],
+    'Forestry': ['Agronomy'],
+    'Textile Science and Engineering': ['Engineering'],
+    'Veterinary Medicine': ['Agronomy'],
+    'Instrument Science and Technology': ['Engineering'],
+    'Physical Education': ['Education']
+}
+
+
+@Benchmark.register(
+    name='super_gpqa',
+    pretty_name='SuperGPQA',
+    dataset_id='m-a-p/SuperGPQA',
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
+    subset_list=SUBSET_LIST,
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='train',  # only have train split
+)
+class SuperGPQAAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        few_shot_num = kwargs.get('few_shot_num', 0)
+        if few_shot_num > 0 and few_shot_num != 5:
+            logger.warning(
+                f'Only support few_shot_num 0 or 5 for SuperGPQA, but got {few_shot_num}. Use 5-shot by default.')
+            kwargs['few_shot_num'] = 5
+        super().__init__(**kwargs)
+
+        self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+        self.category_map = SUBSET_MAPPING
+        self.few_shot_prompt = open(os.path.join(current_dir, 'five_shot_prompt.txt'), encoding='utf-8').read()
+        self.zero_shot_prompt = open(os.path.join(current_dir, 'zero_shot_prompt.txt'), encoding='utf-8').read()
+
+    def load(self, **kwargs):
+        kwargs['subset_list'] = ['default']
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='field', format='{}')
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        if not self.prompt_template:
+            if few_shot_list:
+                prompt = self.few_shot_prompt.format(query=input_d['question'])
+            else:
+                prompt = self.zero_shot_prompt.format(query=input_d['question'])
+        else:
+            prompt = self.prompt_template.format(query=input_d['question'])
+        return self.gen_prompt_data(prompt)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Get the gold choice
+        return input_d.get('answer_letter')
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the model output to get the answer. Could be the best choice index.
+
+        Args:
+            result: Predicted answer from the model. Usually a string for chat.
+            raw_input_d: The raw input. Depending on the dataset.
+            eval_type: 'checkpoint' or 'service' or 'custom'
+
+        Returns:
+            The parsed answer. Depending on the dataset. Usually a string for chat.
+        """
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            from evalscope.benchmarks.super_gpqa.utils import extract_option_content, extract_option_labels
+            sample = raw_input_d
+            if self.few_shot_num == 0:
+                predict = extract_option_labels(result, 'ABCDEFGHIJ')
+                if predict is None:
+                    predict = extract_option_content(result, sample['options'])
+                    predict = chr(sample['options'].index(predict) + 65) if predict else None
+            else:
+                response = result.split('Question:')[0]
+                predict = extract_option_labels(response, 'ABCDEFGHIJ')
+                if predict is None:
+                    predict = extract_option_content(response, sample['options'])
+                    predict = chr(sample['options'].index(predict) + 65) if predict else None
+                if predict is None:
+                    predict = extract_option_labels(result, 'ABCDEFGHIJ')
+                    if predict is None:
+                        predict = extract_option_content(result, sample['options'])
+                        predict = chr(sample['options'].index(predict) + 65) if predict else None
+            return predict
+
+    def match(self, gold: str, pred: str) -> float:
+        return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/super_gpqa/utils.py

@@ -0,0 +1,90 @@
+# flake8: noqa
+import re
+import timeout_decorator
+
+
+@timeout_decorator.timeout(5)  # 5 seconds timeout
+def safe_regex_search(pattern, text, flags=0):
+    try:
+        return re.search(pattern, text, flags)
+    except timeout_decorator.TimeoutError:
+        print(f'Regex match timeout: pattern={pattern}, text={text[:100]}...')
+        return None
+    except Exception as e:
+        print(f'Regex match error: {str(e)}')
+        return None
+
+
+def extract_option_labels(text, options='ABCDEFGHIJ'):
+    if not isinstance(text, str) or not isinstance(options, str):
+        return 'error'
+
+    text = text.rstrip()
+    last_line = text.split('\n')[-1]
+
+    option_str = ''.join([chr(65 + i) for i in range(len(options))]) if options else 'ABCDEFGHIJ'
+
+    patterns = [
+        # e.g. "The final answer to this question is: A."
+        #      "The best option is $\boxed{B}:"
+        #      "The correct answer is (C)."
+        f'[Tt]he\s+(?:\w+\s+)?(?:answer|option)(?:\w+\s+)?\s+is?:?\s*(?:[\*\$\\{{(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*([{option_str}])(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+
+        # e.g. "ANSWER: A"
+        #      "Answer: $\boxed{B}."
+        #      "ANSWER: (C):"
+        f'(?i:Answer)[\*\s]*:\s*(?:[\*\$\\{{(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*([{option_str}])(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+
+        # e.g. "A"
+        #      "$\boxed{B}$"
+        #      "(C)."
+        #      "[D]:"
+        f'^[^\w\r\n]*(?:[\*\$\\{{(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*([{option_str}])(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+    ]
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, last_line, re.IGNORECASE)
+        if match:
+            return match.group(1)
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, text, re.IGNORECASE)
+        if match:
+            return match.group(1)
+
+    return None
+
+
+def extract_option_content(text, options_content=None):
+    if not isinstance(text, str) or not isinstance(options_content, list):
+        return 'error'
+
+    escaped_options_content = [re.escape(option_content) for option_content in options_content]
+    escaped_options_content_str = '|'.join(escaped_options_content)
+
+    text = text.rstrip()
+    last_line = text.split('\n')[-1]
+
+    patterns = [
+        f'[Tt]he\s+(?:\w+\s+)?(?:answer|option)(?:\w+\s+)?\s+is:?\s*(?:[\*\$\\{{\(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*({escaped_options_content_str})(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+        f'(?i:Answer)\s*(?:[\*\$\\{{\(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*({escaped_options_content_str})(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+        f'^[^\w\r\n]*(?:[\*\$\\{{\(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*({escaped_options_content_str})(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+    ]
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, last_line)
+        if match:
+            if match.group(1) in escaped_options_content:
+                return options_content[escaped_options_content.index(match.group(1))]
+            else:
+                return match.group(1)
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, text)
+        if match:
+            if match.group(1) in escaped_options_content:
+                return options_content[escaped_options_content.index(match.group(1))]
+            else:
+                return match.group(1)
+
+    return None

evalscope/benchmarks/super_gpqa/zero_shot_prompt.txt

@@ -0,0 +1,3 @@
+Answer the following multiple choice question. There is only one correct answer. The last line of your response should be in the format 'Answer: $LETTER' (without quotes), where LETTER is one of A, B, C, D, E, F, G, H, I, or J.
+
+{query}

evalscope/benchmarks/trivia_qa/trivia_qa_adapter.py

@@ -5,8 +5,7 @@ import os
 
 from evalscope.benchmarks import Benchmark
 from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.constants import EvalType
-from evalscope.models import ChatGenerationModelAdapter
+from evalscope.constants import EvalType, OutputType
 from evalscope.utils import get_logger
 
 # flake8: noqa
@@ -16,8 +15,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='trivia_qa',
+    pretty_name='TriviaQA',
     dataset_id='modelscope/trivia_qa',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=5,
@@ -100,7 +99,7 @@ class TriviaQaAdapter(DataAdapter):
         context += self._generate_prompt(input_d=input_d, include_answer=False)
         full_prompt = context
 
-        return {'data': [full_prompt], 'system_prompt': prompt or self.prompt_template}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> list:
         # Get the gold choice

evalscope/benchmarks/truthful_qa/truthful_qa_adapter.py

@@ -8,8 +8,7 @@ from typing import List
 
 from evalscope.benchmarks import Benchmark
 from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.constants import EvalType
-from evalscope.models import ContinuationLogitsModelAdapter
+from evalscope.constants import EvalType, OutputType
 from evalscope.utils import get_logger
 
 # flake8: noqa
@@ -21,8 +20,10 @@ logger = get_logger()
 
 @Benchmark.register(
     name='truthful_qa',
+    pretty_name='TruthfulQA',
     dataset_id='modelscope/truthful_qa',
-    model_adapter=ContinuationLogitsModelAdapter,
+    model_adapter=OutputType.CONTINUOUS,
+    output_types=[OutputType.CONTINUOUS, OutputType.GENERATION],
     subset_list=['multiple_choice'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -195,8 +196,7 @@ class TruthfulQaAdapter(DataAdapter):
         else:
             raise ValueError(f'** Unknown subset_name: {subset_name}')
 
-        prompt_d = {'data': ctx_continuation_pair_list}
-        return prompt_d
+        return self.gen_prompt_data(ctx_continuation_pair_list)
 
     def get_gold_answer(self, input_d: dict) -> dict:
         # Get the gold choice
@@ -215,14 +215,7 @@ class TruthfulQaAdapter(DataAdapter):
         Returns:
             The predicted answer.
         """
-        if eval_type == EvalType.CHECKPOINT:
-            return result
-        elif eval_type == EvalType.SERVICE:  # TODO: to be supported !
-            return result
-        elif eval_type == EvalType.CUSTOM:  # TODO: to be supported !
-            return result
-        else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+        return result
 
     def match(self, gold: dict, pred: list) -> dict:
         """
@@ -258,7 +251,7 @@ class TruthfulQaAdapter(DataAdapter):
 
         return {'multiple_choice': {'mc1': mc1(mc1_lls), 'mc2': mc2(mc2_lls)}}  # or {'generation': xxx}
 
-    def compute_metric(self, review_res_list: List[dict]) -> List[dict]:
+    def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]:
         """
         Compute evaluation result by specific metric for each subset.
 

evalscope/benchmarks/utils.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass
+from functools import wraps
+from typing import Dict, List, Optional
+
+from evalscope.constants import EvalType
+from evalscope.utils.filters import Filter
+
+
+@dataclass
+class PromptData:
+    data: List[str]
+    index: Optional[int] = 0
+    system_prompt: Optional[str] = None
+    multi_choices: Optional[List[str]] = None
+
+    def to_dict(self) -> Dict:
+        if self.multi_choices is None:
+            return {
+                'data': self.data,
+                'index': self.index,
+                'system_prompt': self.system_prompt,
+            }
+        else:
+            return {
+                'data': self.data,
+                'index': self.index,
+                'system_prompt': self.system_prompt,
+                'multi_choices': self.multi_choices,
+            }
+
+
+def preprocess_decorator(func):
+
+    @wraps(func)
+    def wrapper(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT):
+        filters = self.config_kwargs.get('filters', None)
+        if filters:
+            # Apply filters to the resultply filters to the result
+            for filter_name, filter_value in filters.items():
+                result = Filter.apply(filter_name, result, filter_value)
+        return func(self, result, raw_input_d, eval_type)
+
+    return wrapper

evalscope/cli/start_app.py

@@ -3,7 +3,6 @@ import os
 from argparse import ArgumentParser
 
 from evalscope.cli.base import CLICommand
-from evalscope.report.app import add_argument, create_app
 
 
 def subparser_func(args):
@@ -22,9 +21,13 @@ class StartAppCMD(CLICommand):
     def define_args(parsers: ArgumentParser):
         """ define args for create pipeline template command.
         """
+        from evalscope.report.app import add_argument
+
         parser = parsers.add_parser(StartAppCMD.name)
         add_argument(parser)
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
+        from evalscope.report.app import create_app
+
         create_app(self.args)

evalscope/cli/start_eval.py

@@ -1,10 +1,7 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os
 from argparse import ArgumentParser
 
-from evalscope.arguments import add_argument
 from evalscope.cli.base import CLICommand
-from evalscope.run import run_task
 
 
 def subparser_func(args):
@@ -23,9 +20,13 @@ class EvalCMD(CLICommand):
     def define_args(parsers: ArgumentParser):
         """ define args for create pipeline template command.
         """
+        from evalscope.arguments import add_argument
+
         parser = parsers.add_parser(EvalCMD.name)
         add_argument(parser)
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
+        from evalscope.run import run_task
+
         run_task(self.args)

evalscope/cli/start_perf.py

@@ -3,8 +3,6 @@ import os
 from argparse import ArgumentParser
 
 from evalscope.cli.base import CLICommand
-from evalscope.perf.arguments import add_argument
-from evalscope.perf.main import run_perf_benchmark
 
 
 def subparser_func(args):
@@ -23,9 +21,13 @@ class PerfBenchCMD(CLICommand):
     def define_args(parsers: ArgumentParser):
         """ define args for create pipeline template command.
         """
+        from evalscope.perf.arguments import add_argument
+
         parser = parsers.add_parser(PerfBenchCMD.name)
         add_argument(parser)
         parser.set_defaults(func=subparser_func)
 
     def execute(self):
+        from evalscope.perf.main import run_perf_benchmark
+
         run_perf_benchmark(self.args)

evalscope/collections/evaluator.py

@@ -97,8 +97,8 @@ class EvaluatorCollection:
         evaluators = {}
         for dataset_name in self.dataset_name_map.keys():
             benchmark = Benchmark.get(dataset_name)
+            model_adapter = initialize_model_adapter(self.task_cfg, benchmark, self.model)
             data_adapter = benchmark.get_data_adapter()
-            model_adapter = initialize_model_adapter(self.task_cfg, benchmark.model_adapter, self.model)
             evaluators[dataset_name] = SimpleEvaluator(dataset_name, data_adapter, model_adapter, self.task_cfg,
                                                        self.outputs)
         return evaluators
@@ -234,6 +234,21 @@ class EvaluatorCollection:
     def get_reviews(self, answers):
         review_file_path = os.path.join(self.outputs.reviews_dir, self.task_cfg.model_id)
         os.makedirs(review_file_path, exist_ok=True)
+
+        if self.task_cfg.use_cache and os.path.exists(review_file_path):
+            logger.warning(
+                f'Ignore use_cache={self.task_cfg.use_cache}, updating the review file: {review_file_path} ...')
+            if os.path.isdir(review_file_path):
+                for filename in os.listdir(review_file_path):
+                    file_path = os.path.join(review_file_path, filename)
+                    try:
+                        if os.path.isfile(file_path):
+                            os.remove(file_path)
+                    except Exception as e:
+                        logger.error(f'Error deleting file {file_path}: {e}')
+            else:
+                os.remove(review_file_path)
+
         reviews = defaultdict(dict)
         for sample in tqdm(self.dataset, desc='Getting reviews'):
             evaluator = self.evaluators[sample.dataset_name]

evalscope/config.py

@@ -4,10 +4,12 @@ import copy
 import json
 import os
 from argparse import Namespace
+from collections import OrderedDict
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Union
 
-from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, DEFAULT_WORK_DIR, EvalBackend, EvalStage, EvalType, HubType
+from evalscope.constants import (DEFAULT_DATASET_CACHE_DIR, DEFAULT_WORK_DIR, EvalBackend, EvalStage, EvalType, HubType,
+                                 OutputType)
 from evalscope.models.custom import CustomModel
 from evalscope.utils import gen_hash
 from evalscope.utils.io_utils import dict_to_yaml, json_to_dict, yaml_to_dict
@@ -17,7 +19,7 @@ logger = get_logger()
 
 cur_path = os.path.dirname(os.path.abspath(__file__))
 
-DEFAULT_MODEL_ARGS = {'revision': 'master', 'precision': 'torch.float16', 'device': 'auto'}
+DEFAULT_MODEL_ARGS = {'revision': 'master', 'precision': 'torch.float16'}
 DEFAULT_GENERATION_CONFIG = {
     'max_length': 2048,
     'max_new_tokens': 512,
@@ -54,7 +56,7 @@ class TaskConfig:
     eval_config: Union[str, Dict, None] = None
     stage: str = EvalStage.ALL
     limit: Optional[int] = None
-    eval_batch_size: int = 1
+    eval_batch_size: Optional[int] = None
 
     # Cache and working directory arguments
     mem_cache: bool = False  # Deprecated, will be removed in v1.0.0.
@@ -68,6 +70,8 @@ class TaskConfig:
     seed: Optional[int] = 42
     api_url: Optional[str] = None  # Only used for server model
     api_key: Optional[str] = 'EMPTY'  # Only used for server model
+    timeout: Optional[float] = None  # Only used for server model
+    stream: bool = False  # Only used for server model
 
     def __post_init__(self):
         if (not self.model_id) and self.model:
@@ -75,6 +79,12 @@ class TaskConfig:
                 self.model_id = type(self.model).__name__
             else:
                 self.model_id = os.path.basename(self.model).rstrip(os.sep)
+            # fix path error, see http://github.com/modelscope/evalscope/issues/377
+            self.model_id = self.model_id.replace(':', '-')
+
+        # Set default eval_batch_size based on eval_type
+        if self.eval_batch_size is None:
+            self.eval_batch_size = 8 if self.eval_type == EvalType.SERVICE else 1
 
     def to_dict(self):
         return self.__dict__

evalscope/constants.py

@@ -139,6 +139,13 @@ class EvalType:
     SERVICE = 'service'  # model service
 
 
+class OutputType:
+    LOGITS = 'logits'  # for multiple choice tasks
+    GENERATION = 'generation'  # for text generation tasks and general tasks
+    MULTIPLE_CHOICE = 'multiple_choice_logits'  # for multiple choice tasks
+    CONTINUOUS = 'continuous_logits'  # for continuous tasks
+
+
 class EvalBackend:
     NATIVE = 'Native'
     OPEN_COMPASS = 'OpenCompass'

evalscope/evaluator/evaluator.py

@@ -250,6 +250,7 @@ class Evaluator(object):
 
         if self.use_cache and os.path.exists(review_file_path):
             logger.warning(f'Ignore use_cache={self.use_cache}, updating the review file: {review_file_path} ...')
+            os.remove(review_file_path)
 
         for answer_d in tqdm(answers_list, total=len(answers_list), desc=f'Reviewing({subset_name}): '):
             review_id, reviewer_spec = self._generate_review_id(answer_d)
@@ -292,7 +293,8 @@ class Evaluator(object):
 
             review_res_list.append(review_res)
 
-        metric_score: List[dict] = self.data_adapter.compute_metric(review_res_list=review_res_list)
+        metric_score: List[dict] = self.data_adapter.compute_metric(
+            review_res_list=review_res_list, reviews_list=reviews_list)
 
         return metric_score
 

evalscope/metrics/__init__.py

@@ -1,4 +1,5 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-from evalscope.metrics.metrics import bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean, weighted_mean
+from evalscope.metrics.metrics import (bleu_ngram_one_sample, exact_match, macro_mean, mean, micro_mean,
+                                       simple_f1_score, weighted_mean)
 from evalscope.metrics.named_metrics import *
 from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh

evalscope/metrics/metrics.py

@@ -13,6 +13,9 @@ from typing import TYPE_CHECKING, Dict, List, Union
 
 
 def mean(arr: list):
+    if not arr:
+        return 0.0
+
     if isinstance(arr[0], list):
         arr = [item for sublist in arr for item in sublist]
     return sum(arr) / len(arr)
@@ -58,6 +61,18 @@ def matthews_corrcoef(items):
     return sklearn.metrics.matthews_corrcoef(golds, preds)
 
 
+def simple_f1_score(scores: tuple) -> float:
+    score1 = scores[0]
+    score2 = scores[1]
+    score1 = np.mean(score1) if len(score1) > 0 else 0.0
+    score2 = np.mean(score2) if len(score2) > 0 else 0.0
+
+    if score1 == 0 and score2 == 0:
+        return 0.0
+    else:
+        return 2 * score1 * score2 / (score1 + score2)
+
+
 def f1_score(items):
     import sklearn.metrics
 
@@ -126,11 +141,17 @@ def weighted_mean(items: List) -> float:
 
 
 def micro_mean(items):
-    return sum([item.score * item.num for item in items]) / sum([item.num for item in items])
+    try:
+        return sum([item.score * item.num for item in items]) / sum([item.num for item in items])
+    except ZeroDivisionError:
+        return 0.0
 
 
 def macro_mean(items):
-    return sum([item.score for item in items]) / len(items)
+    try:
+        return sum([item.score for item in items]) / len(items)
+    except ZeroDivisionError:
+        return 0.0
 
 
 def weighted_perplexity(items):