evalscope 0.12.0__py3-none-any.whl → 0.13.0__py3-none-any.whl

evalscope/benchmarks/super_gpqa/super_gpqa_adapter.py

@@ -0,0 +1,191 @@
+import os
+import random
+import re
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType, OutputType
+from evalscope.metrics import exact_match
+from evalscope.utils import logger
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+
+SUBSET_LIST = [
+    'Electronic Science and Technology', 'Philosophy', 'Traditional Chinese Medicine', 'Applied Economics',
+    'Mathematics', 'Physics', 'Clinical Medicine', 'Computer Science and Technology',
+    'Information and Communication Engineering', 'Control Science and Engineering', 'Theoretical Economics', 'Law',
+    'History', 'Basic Medicine', 'Education', 'Materials Science and Engineering', 'Electrical Engineering',
+    'Systems Science', 'Power Engineering and Engineering Thermophysics', 'Military Science', 'Biology',
+    'Business Administration', 'Language and Literature', 'Public Health and Preventive Medicine', 'Political Science',
+    'Chemistry', 'Hydraulic Engineering', 'Chemical Engineering and Technology', 'Pharmacy', 'Geography', 'Art Studies',
+    'Architecture', 'Forestry Engineering', 'Public Administration', 'Oceanography', 'Journalism and Communication',
+    'Nuclear Science and Technology', 'Weapon Science and Technology', 'Naval Architecture and Ocean Engineering',
+    'Environmental Science and Engineering', 'Transportation Engineering', 'Geology', 'Physical Oceanography',
+    'Musicology', 'Stomatology', 'Aquaculture', 'Mechanical Engineering',
+    'Aeronautical and Astronautical Science and Technology', 'Civil Engineering', 'Mechanics',
+    'Petroleum and Natural Gas Engineering', 'Sociology', 'Food Science and Engineering', 'Agricultural Engineering',
+    'Surveying and Mapping Science and Technology', 'Metallurgical Engineering',
+    'Library, Information and Archival Management', 'Mining Engineering', 'Astronomy',
+    'Geological Resources and Geological Engineering', 'Atmospheric Science', 'Optical Engineering', 'Animal Husbandry',
+    'Geophysics', 'Crop Science', 'Management Science and Engineering', 'Psychology', 'Forestry',
+    'Textile Science and Engineering', 'Veterinary Medicine', 'Instrument Science and Technology', 'Physical Education'
+]
+
+SUBSET_MAPPING = {
+    'Electronic Science and Technology': ['Engineering'],
+    'Philosophy': ['Philosophy'],
+    'Traditional Chinese Medicine': ['Medicine'],
+    'Applied Economics': ['Economics'],
+    'Mathematics': ['Science'],
+    'Physics': ['Science'],
+    'Clinical Medicine': ['Medicine'],
+    'Computer Science and Technology': ['Engineering'],
+    'Information and Communication Engineering': ['Engineering'],
+    'Control Science and Engineering': ['Engineering'],
+    'Theoretical Economics': ['Economics'],
+    'Law': ['Law'],
+    'History': ['History'],
+    'Basic Medicine': ['Medicine'],
+    'Education': ['Education'],
+    'Materials Science and Engineering': ['Engineering'],
+    'Electrical Engineering': ['Engineering'],
+    'Systems Science': ['Science'],
+    'Power Engineering and Engineering Thermophysics': ['Engineering'],
+    'Military Science': ['Military Science'],
+    'Biology': ['Science'],
+    'Business Administration': ['Management'],
+    'Language and Literature': ['Literature and Arts'],
+    'Public Health and Preventive Medicine': ['Medicine'],
+    'Political Science': ['Law'],
+    'Chemistry': ['Science'],
+    'Hydraulic Engineering': ['Engineering'],
+    'Chemical Engineering and Technology': ['Engineering'],
+    'Pharmacy': ['Medicine'],
+    'Geography': ['Science'],
+    'Art Studies': ['Literature and Arts'],
+    'Architecture': ['Engineering'],
+    'Forestry Engineering': ['Engineering'],
+    'Public Administration': ['Management'],
+    'Oceanography': ['Science'],
+    'Journalism and Communication': ['Literature and Arts'],
+    'Nuclear Science and Technology': ['Engineering'],
+    'Weapon Science and Technology': ['Engineering'],
+    'Naval Architecture and Ocean Engineering': ['Engineering'],
+    'Environmental Science and Engineering': ['Engineering'],
+    'Transportation Engineering': ['Engineering'],
+    'Geology': ['Science'],
+    'Physical Oceanography': ['Science'],
+    'Musicology': ['Literature and Arts'],
+    'Stomatology': ['Medicine'],
+    'Aquaculture': ['Agronomy'],
+    'Mechanical Engineering': ['Engineering'],
+    'Aeronautical and Astronautical Science and Technology': ['Engineering'],
+    'Civil Engineering': ['Engineering'],
+    'Mechanics': ['Engineering'],
+    'Petroleum and Natural Gas Engineering': ['Engineering'],
+    'Sociology': ['Sociology'],
+    'Food Science and Engineering': ['Engineering'],
+    'Agricultural Engineering': ['Engineering'],
+    'Surveying and Mapping Science and Technology': ['Engineering'],
+    'Metallurgical Engineering': ['Engineering'],
+    'Library, Information and Archival Management': ['Management'],
+    'Mining Engineering': ['Engineering'],
+    'Astronomy': ['Science'],
+    'Geological Resources and Geological Engineering': ['Engineering'],
+    'Atmospheric Science': ['Science'],
+    'Optical Engineering': ['Engineering'],
+    'Animal Husbandry': ['Agronomy'],
+    'Geophysics': ['Science'],
+    'Crop Science': ['Agronomy'],
+    'Management Science and Engineering': ['Management'],
+    'Psychology': ['Education'],
+    'Forestry': ['Agronomy'],
+    'Textile Science and Engineering': ['Engineering'],
+    'Veterinary Medicine': ['Agronomy'],
+    'Instrument Science and Technology': ['Engineering'],
+    'Physical Education': ['Education']
+}
+
+
+@Benchmark.register(
+    name='super_gpqa',
+    pretty_name='SuperGPQA',
+    dataset_id='m-a-p/SuperGPQA',
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
+    subset_list=SUBSET_LIST,
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='train',  # only have train split
+)
+class SuperGPQAAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        few_shot_num = kwargs.get('few_shot_num', 0)
+        if few_shot_num > 0 and few_shot_num != 5:
+            logger.warning(
+                f'Only support few_shot_num 0 or 5 for SuperGPQA, but got {few_shot_num}. Use 5-shot by default.')
+            kwargs['few_shot_num'] = 5
+        super().__init__(**kwargs)
+
+        self.choices = ['A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J']
+        self.category_map = SUBSET_MAPPING
+        self.few_shot_prompt = open(os.path.join(current_dir, 'five_shot_prompt.txt'), encoding='utf-8').read()
+        self.zero_shot_prompt = open(os.path.join(current_dir, 'zero_shot_prompt.txt'), encoding='utf-8').read()
+
+    def load(self, **kwargs):
+        kwargs['subset_list'] = ['default']
+        data_dict = super().load(**kwargs)
+        return self.reformat_subset(data_dict, subset_key='field', format='{}')
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        if not self.prompt_template:
+            if few_shot_list:
+                prompt = self.few_shot_prompt.format(query=input_d['question'])
+            else:
+                prompt = self.zero_shot_prompt.format(query=input_d['question'])
+        else:
+            prompt = self.prompt_template.format(query=input_d['question'])
+        return self.gen_prompt_data(prompt)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Get the gold choice
+        return input_d.get('answer_letter')
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the model output to get the answer. Could be the best choice index.
+
+        Args:
+            result: Predicted answer from the model. Usually a string for chat.
+            raw_input_d: The raw input. Depending on the dataset.
+            eval_type: 'checkpoint' or 'service' or 'custom'
+
+        Returns:
+            The parsed answer. Depending on the dataset. Usually a string for chat.
+        """
+        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
+            return result
+        else:
+            from evalscope.benchmarks.super_gpqa.utils import extract_option_content, extract_option_labels
+            sample = raw_input_d
+            if self.few_shot_num == 0:
+                predict = extract_option_labels(result, 'ABCDEFGHIJ')
+                if predict is None:
+                    predict = extract_option_content(result, sample['options'])
+                    predict = chr(sample['options'].index(predict) + 65) if predict else None
+            else:
+                response = result.split('Question:')[0]
+                predict = extract_option_labels(response, 'ABCDEFGHIJ')
+                if predict is None:
+                    predict = extract_option_content(response, sample['options'])
+                    predict = chr(sample['options'].index(predict) + 65) if predict else None
+                if predict is None:
+                    predict = extract_option_labels(result, 'ABCDEFGHIJ')
+                    if predict is None:
+                        predict = extract_option_content(result, sample['options'])
+                        predict = chr(sample['options'].index(predict) + 65) if predict else None
+            return predict
+
+    def match(self, gold: str, pred: str) -> float:
+        return exact_match(gold=gold, pred=pred)

evalscope/benchmarks/super_gpqa/utils.py

@@ -0,0 +1,85 @@
+# flake8: noqa
+import re
+
+
+def safe_regex_search(pattern, text, flags=0):
+    try:
+        return re.search(pattern, text, flags)
+    except Exception as e:
+        print(f'Regex match error: {str(e)}')
+        return None
+
+
+def extract_option_labels(text, options='ABCDEFGHIJ'):
+    if not isinstance(text, str) or not isinstance(options, str):
+        return 'error'
+
+    text = text.rstrip()
+    last_line = text.split('\n')[-1]
+
+    option_str = ''.join([chr(65 + i) for i in range(len(options))]) if options else 'ABCDEFGHIJ'
+
+    patterns = [
+        # e.g. "The final answer to this question is: A."
+        #      "The best option is $\boxed{B}:"
+        #      "The correct answer is (C)."
+        f'[Tt]he\s+(?:\w+\s+)?(?:answer|option)(?:\w+\s+)?\s+is?:?\s*(?:[\*\$\\{{(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*([{option_str}])(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+
+        # e.g. "ANSWER: A"
+        #      "Answer: $\boxed{B}."
+        #      "ANSWER: (C):"
+        f'(?i:Answer)[\*\s]*:\s*(?:[\*\$\\{{(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*([{option_str}])(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+
+        # e.g. "A"
+        #      "$\boxed{B}$"
+        #      "(C)."
+        #      "[D]:"
+        f'^[^\w\r\n]*(?:[\*\$\\{{(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*([{option_str}])(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+    ]
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, last_line, re.IGNORECASE)
+        if match:
+            return match.group(1)
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, text, re.IGNORECASE)
+        if match:
+            return match.group(1)
+
+    return None
+
+
+def extract_option_content(text, options_content=None):
+    if not isinstance(text, str) or not isinstance(options_content, list):
+        return 'error'
+
+    escaped_options_content = [re.escape(option_content) for option_content in options_content]
+    escaped_options_content_str = '|'.join(escaped_options_content)
+
+    text = text.rstrip()
+    last_line = text.split('\n')[-1]
+
+    patterns = [
+        f'[Tt]he\s+(?:\w+\s+)?(?:answer|option)(?:\w+\s+)?\s+is:?\s*(?:[\*\$\\{{\(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*({escaped_options_content_str})(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+        f'(?i:Answer)\s*(?:[\*\$\\{{\(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*({escaped_options_content_str})(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+        f'^[^\w\r\n]*(?:[\*\$\\{{\(\[\\\\(]*?(?:(?:\\\\boxed|\\\\mathbf|\\\\mathrm|\\\\text){{)?)*\s*({escaped_options_content_str})(?:\\\\?\}}?\$?\)?\]?\}}?)*(?:[\s:\.\*)]|$)',
+    ]
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, last_line)
+        if match:
+            if match.group(1) in escaped_options_content:
+                return options_content[escaped_options_content.index(match.group(1))]
+            else:
+                return match.group(1)
+
+    for pattern in patterns:
+        match = safe_regex_search(pattern, text)
+        if match:
+            if match.group(1) in escaped_options_content:
+                return options_content[escaped_options_content.index(match.group(1))]
+            else:
+                return match.group(1)
+
+    return None

evalscope/benchmarks/super_gpqa/zero_shot_prompt.txt

@@ -0,0 +1,3 @@
+Answer the following multiple choice question. There is only one correct answer. The last line of your response should be in the format 'Answer: $LETTER' (without quotes), where LETTER is one of A, B, C, D, E, F, G, H, I, or J.
+
+{query}

evalscope/benchmarks/trivia_qa/trivia_qa_adapter.py

@@ -5,8 +5,7 @@ import os
 
 from evalscope.benchmarks import Benchmark
 from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.constants import EvalType
-from evalscope.models import ChatGenerationModelAdapter
+from evalscope.constants import EvalType, OutputType
 from evalscope.utils import get_logger
 
 # flake8: noqa
@@ -16,8 +15,8 @@ logger = get_logger()
 
 @Benchmark.register(
     name='trivia_qa',
+    pretty_name='TriviaQA',
     dataset_id='modelscope/trivia_qa',
-    model_adapter=ChatGenerationModelAdapter,
     subset_list=['default'],
     metric_list=['AverageAccuracy'],
     few_shot_num=5,
@@ -100,7 +99,7 @@ class TriviaQaAdapter(DataAdapter):
         context += self._generate_prompt(input_d=input_d, include_answer=False)
         full_prompt = context
 
-        return {'data': [full_prompt], 'system_prompt': prompt or self.prompt_template}
+        return self.gen_prompt_data(full_prompt)
 
     def get_gold_answer(self, input_d: dict) -> list:
         # Get the gold choice

evalscope/benchmarks/truthful_qa/truthful_qa_adapter.py

@@ -8,8 +8,7 @@ from typing import List
 
 from evalscope.benchmarks import Benchmark
 from evalscope.benchmarks.data_adapter import DataAdapter
-from evalscope.constants import EvalType
-from evalscope.models import ContinuationLogitsModelAdapter
+from evalscope.constants import EvalType, OutputType
 from evalscope.utils import get_logger
 
 # flake8: noqa
@@ -21,8 +20,10 @@ logger = get_logger()
 
 @Benchmark.register(
     name='truthful_qa',
+    pretty_name='TruthfulQA',
     dataset_id='modelscope/truthful_qa',
-    model_adapter=ContinuationLogitsModelAdapter,
+    model_adapter=OutputType.CONTINUOUS,
+    output_types=[OutputType.CONTINUOUS, OutputType.GENERATION],
     subset_list=['multiple_choice'],
     metric_list=['AverageAccuracy'],
     few_shot_num=0,
@@ -195,8 +196,7 @@ class TruthfulQaAdapter(DataAdapter):
         else:
             raise ValueError(f'** Unknown subset_name: {subset_name}')
 
-        prompt_d = {'data': ctx_continuation_pair_list}
-        return prompt_d
+        return self.gen_prompt_data(ctx_continuation_pair_list)
 
     def get_gold_answer(self, input_d: dict) -> dict:
         # Get the gold choice
@@ -215,14 +215,7 @@ class TruthfulQaAdapter(DataAdapter):
         Returns:
             The predicted answer.
         """
-        if eval_type == EvalType.CHECKPOINT:
-            return result
-        elif eval_type == EvalType.SERVICE:  # TODO: to be supported !
-            return result
-        elif eval_type == EvalType.CUSTOM:  # TODO: to be supported !
-            return result
-        else:
-            raise ValueError(f'Invalid eval_type: {eval_type}')
+        return result
 
     def match(self, gold: dict, pred: list) -> dict:
         """

evalscope/benchmarks/utils.py

@@ -0,0 +1,43 @@
+from dataclasses import dataclass
+from functools import wraps
+from typing import Dict, List, Optional
+
+from evalscope.constants import EvalType
+from evalscope.utils.filters import Filter
+
+
+@dataclass
+class PromptData:
+    data: List[str]
+    index: Optional[int] = 0
+    system_prompt: Optional[str] = None
+    multi_choices: Optional[List[str]] = None
+
+    def to_dict(self) -> Dict:
+        if self.multi_choices is None:
+            return {
+                'data': self.data,
+                'index': self.index,
+                'system_prompt': self.system_prompt,
+            }
+        else:
+            return {
+                'data': self.data,
+                'index': self.index,
+                'system_prompt': self.system_prompt,
+                'multi_choices': self.multi_choices,
+            }
+
+
+def preprocess_decorator(func):
+
+    @wraps(func)
+    def wrapper(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT):
+        filters = self.config_kwargs.get('filters', None)
+        if filters:
+            # Apply filters to the resultply filters to the result
+            for filter_name, filter_value in filters.items():
+                result = Filter.apply(filter_name, result, filter_value)
+        return func(self, result, raw_input_d, eval_type)
+
+    return wrapper

evalscope/collections/evaluator.py

@@ -12,7 +12,7 @@ from evalscope.collections.sampler import DatasetEntry
 from evalscope.config import TaskConfig
 from evalscope.constants import AnswerKeys, DumpMode, EvalType
 from evalscope.evaluator import Evaluator
-from evalscope.models import get_local_model, initialize_model_adapter
+from evalscope.models import initialize_model_adapter
 from evalscope.report import ReportGenerator
 from evalscope.utils.io_utils import OutputsStructure, dump_jsonl_data, jsonl_to_list
 from evalscope.utils.logger import get_logger
@@ -53,11 +53,11 @@ class SimpleEvaluator(Evaluator):
 
 class EvaluatorCollection:
 
-    def __init__(self, task_cfg: TaskConfig, data_adapter: DataAdapter, outputs: OutputsStructure):
+    def __init__(self, task_cfg: TaskConfig, data_adapter: DataAdapter, outputs: OutputsStructure, base_model):
         self.task_cfg = task_cfg
         self.data_adapter = data_adapter
         self.outputs = outputs
-        self.model = get_local_model(task_cfg)
+        self.model = base_model
 
         self.dataset, self.dataset_name = self.load()
         self.dataset_name_map = EvaluatorCollection._init_name_map(self.dataset)
@@ -97,8 +97,8 @@ class EvaluatorCollection:
         evaluators = {}
         for dataset_name in self.dataset_name_map.keys():
             benchmark = Benchmark.get(dataset_name)
+            model_adapter = initialize_model_adapter(self.task_cfg, benchmark, self.model)
             data_adapter = benchmark.get_data_adapter()
-            model_adapter = initialize_model_adapter(self.task_cfg, benchmark.model_adapter, self.model)
             evaluators[dataset_name] = SimpleEvaluator(dataset_name, data_adapter, model_adapter, self.task_cfg,
                                                        self.outputs)
         return evaluators
@@ -238,7 +238,16 @@ class EvaluatorCollection:
         if self.task_cfg.use_cache and os.path.exists(review_file_path):
             logger.warning(
                 f'Ignore use_cache={self.task_cfg.use_cache}, updating the review file: {review_file_path} ...')
-            os.remove(review_file_path)
+            if os.path.isdir(review_file_path):
+                for filename in os.listdir(review_file_path):
+                    file_path = os.path.join(review_file_path, filename)
+                    try:
+                        if os.path.isfile(file_path):
+                            os.remove(file_path)
+                    except Exception as e:
+                        logger.error(f'Error deleting file {file_path}: {e}')
+            else:
+                os.remove(review_file_path)
 
         reviews = defaultdict(dict)
         for sample in tqdm(self.dataset, desc='Getting reviews'):

evalscope/config.py

@@ -4,10 +4,12 @@ import copy
 import json
 import os
 from argparse import Namespace
+from collections import OrderedDict
 from dataclasses import dataclass, field
 from typing import Dict, List, Optional, Union
 
-from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, DEFAULT_WORK_DIR, EvalBackend, EvalStage, EvalType, HubType
+from evalscope.constants import (DEFAULT_DATASET_CACHE_DIR, DEFAULT_WORK_DIR, EvalBackend, EvalStage, EvalType, HubType,
+                                 JudgeStrategy, OutputType)
 from evalscope.models.custom import CustomModel
 from evalscope.utils import gen_hash
 from evalscope.utils.io_utils import dict_to_yaml, json_to_dict, yaml_to_dict
@@ -54,7 +56,7 @@ class TaskConfig:
     eval_config: Union[str, Dict, None] = None
     stage: str = EvalStage.ALL
     limit: Optional[int] = None
-    eval_batch_size: int = 1
+    eval_batch_size: Optional[int] = None
 
     # Cache and working directory arguments
     mem_cache: bool = False  # Deprecated, will be removed in v1.0.0.
@@ -71,12 +73,23 @@ class TaskConfig:
     timeout: Optional[float] = None  # Only used for server model
     stream: bool = False  # Only used for server model
 
+    # LLMJudge arguments
+    judge_strategy: str = JudgeStrategy.AUTO
+    judge_worker_num: int = 8
+    judge_model_args: Optional[Dict] = field(default_factory=lambda: {})
+
     def __post_init__(self):
         if (not self.model_id) and self.model:
             if isinstance(self.model, CustomModel):
                 self.model_id = type(self.model).__name__
             else:
                 self.model_id = os.path.basename(self.model).rstrip(os.sep)
+            # fix path error, see http://github.com/modelscope/evalscope/issues/377
+            self.model_id = self.model_id.replace(':', '-')
+
+        # Set default eval_batch_size based on eval_type
+        if self.eval_batch_size is None:
+            self.eval_batch_size = 8 if self.eval_type == EvalType.SERVICE else 1
 
     def to_dict(self):
         return self.__dict__

evalscope/constants.py

@@ -139,6 +139,13 @@ class EvalType:
     SERVICE = 'service'  # model service
 
 
+class OutputType:
+    LOGITS = 'logits'  # for multiple choice tasks
+    GENERATION = 'generation'  # for text generation tasks and general tasks
+    MULTIPLE_CHOICE = 'multiple_choice_logits'  # for multiple choice tasks
+    CONTINUOUS = 'continuous_logits'  # for continuous tasks
+
+
 class EvalBackend:
     NATIVE = 'Native'
     OPEN_COMPASS = 'OpenCompass'
@@ -149,3 +156,10 @@ class EvalBackend:
 
 class DataCollection:
     NAME = 'data_collection'
+
+
+class JudgeStrategy:
+    AUTO = 'auto'
+    RULE = 'rule'
+    LLM = 'llm'
+    LLM_RECALL = 'llm_recall'

evalscope/evaluator/evaluator.py

@@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Union
 
 from evalscope.benchmarks import DataAdapter
 from evalscope.config import TaskConfig
-from evalscope.constants import AnswerKeys, DumpMode, EvalStage, EvalType, ReviewKeys
+from evalscope.constants import AnswerKeys, DumpMode, EvalStage, EvalType, JudgeStrategy, ReviewKeys
 from evalscope.models import BaseModelAdapter
 from evalscope.report import Report, gen_table
 from evalscope.utils import dict_torch_dtype_to_str, gen_hash
@@ -58,9 +58,17 @@ class Evaluator(object):
         self.task_cfg = task_cfg
         # Deal with the output paths
         self.outputs_structure = outputs
-
         self.kwargs = kwargs
 
+        self._init_judge()
+
+    def _init_judge(self):
+        if self.task_cfg.judge_strategy == JudgeStrategy.RULE:
+            self.judge = None
+        else:
+            from evalscope.metrics.llm_judge import LLMJudge
+            self.judge = LLMJudge(**self.task_cfg.judge_model_args)
+
     def load_dataset(self):
         dataset = self.data_adapter.load(
             work_dir=os.path.expanduser(self.task_cfg.dataset_dir), datasets_hub=self.dataset_hub, **self.kwargs)
@@ -200,17 +208,40 @@ class Evaluator(object):
         for choice in choices:
             raw_input_d: dict = review_res[AnswerKeys.RAW_INPUT]
             answer_content = choice[ReviewKeys.MESSAGE][ReviewKeys.CONTENT]
-            answer_content = self.data_adapter.parse_pred_result(
-                result=answer_content, raw_input_d=raw_input_d, eval_type=self.eval_type)
             gold_content = self.data_adapter.get_gold_answer(raw_input_d)
 
-            review_result = self.data_adapter.match(gold_content, answer_content)
+            # Get review result based on judge strategy
+            use_llm = (
+                self.task_cfg.judge_strategy == JudgeStrategy.LLM
+                or (self.task_cfg.judge_strategy == JudgeStrategy.AUTO and self.data_adapter.llm_as_a_judge))
+
+            if use_llm:
+                # Use LLM as judge
+                assert self.judge is not None, f'Judge model is required for LLM judging {self.data_adapter.name}'
+                review_result = self.data_adapter.llm_match(
+                    gold_content, answer_content, self.judge, raw_input=raw_input_d)
+                pred = answer_content
+            else:
+                # Use rule-based judging
+                pred_content = self.data_adapter.parse_pred_result(
+                    result=answer_content, raw_input_d=raw_input_d, eval_type=self.eval_type)
+                review_result = self.data_adapter.match(gold_content, pred_content)
+
+                # For LLM_RECALL strategy, use LLM to re-judge if rule-based result is not good
+                if (self.task_cfg.judge_strategy == JudgeStrategy.LLM_RECALL
+                        and isinstance(review_result, (bool, int, float)) and not bool(review_result)):
+                    assert self.judge is not None, f'Judge model is required for LLM_RECALL strategy {self.data_adapter.name}'  # noqa: E501
+                    review_result = self.data_adapter.llm_match(
+                        gold_content, answer_content, self.judge, raw_input=raw_input_d)
+                    pred = answer_content
+                else:
+                    pred = pred_content
+
             choice[ReviewKeys.REVIEW] = {
                 ReviewKeys.GOLD: gold_content,
-                ReviewKeys.PRED: answer_content,
+                ReviewKeys.PRED: pred,
                 ReviewKeys.RESULT: review_result
             }
-
             rev_choices.append(choice)
 
         review_res[AnswerKeys.CHOICES] = rev_choices
@@ -252,16 +283,23 @@ class Evaluator(object):
             logger.warning(f'Ignore use_cache={self.use_cache}, updating the review file: {review_file_path} ...')
             os.remove(review_file_path)
 
-        for answer_d in tqdm(answers_list, total=len(answers_list), desc=f'Reviewing({subset_name}): '):
+        def process_single_review(answer_d):
             review_id, reviewer_spec = self._generate_review_id(answer_d)
             # Get review
             review_d = self._get_review(answer_d=answer_d, review_id=review_id, reviewer_spec=reviewer_spec)
-
             logger.debug(review_d)
-
-            reviews_list.append(review_d)
-            # Dump reviews
-            dump_jsonl_data(review_d, review_file_path, dump_mode=DumpMode.APPEND)
+            return review_d
+
+        with ThreadPoolExecutor(max_workers=self.task_cfg.judge_worker_num) as executor:
+            # Submit all tasks and get futures
+            futures = [executor.submit(process_single_review, answer_d) for answer_d in answers_list]
+
+            # Process completed futures with progress bar
+            for future in tqdm(as_completed(futures), total=len(futures), desc=f'Reviewing({subset_name}): '):
+                review_d = future.result()
+                reviews_list.append(review_d)
+                # Dump reviews
+                dump_jsonl_data(review_d, review_file_path, dump_mode=DumpMode.APPEND)
 
         return reviews_list