evalscope 0.14.0__py3-none-any.whl → 0.15.1__py3-none-any.whl

evalscope/arguments.py

@@ -1,7 +1,7 @@
 import argparse
 import json
 
-from evalscope.constants import EvalBackend, EvalStage, EvalType, JudgeStrategy, OutputType
+from evalscope.constants import EvalBackend, EvalStage, EvalType, JudgeStrategy, ModelTask, OutputType
 
 
 class ParseStrArgsAction(argparse.Action):
@@ -35,6 +35,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--model', type=str, required=False, help='The model id on modelscope, or local model dir.')
     parser.add_argument('--model-id', type=str, required=False, help='The model id for model name in report.')
     parser.add_argument('--model-args', type=str, action=ParseStrArgsAction, help='The model args, should be a string.')
+    parser.add_argument('--model-task', type=str, default=ModelTask.TEXT_GENERATION, choices=[ModelTask.TEXT_GENERATION, ModelTask.IMAGE_GENERATION], help='The model task for model id.')  # noqa: E501
 
     # Template-related arguments
     parser.add_argument('--template-type', type=str, required=False, help='Deprecated, will be removed in v1.0.0.')

evalscope/benchmarks/__init__.py

@@ -10,8 +10,8 @@ from evalscope.utils import get_logger
 logger = get_logger()
 
 # Using glob to find all files matching the pattern
-pattern = os.path.join(os.path.dirname(__file__), '*', '*_adapter.py')
-files = glob.glob(pattern, recursive=False)
+pattern = os.path.join(os.path.dirname(__file__), '*', '**', '*_adapter.py')
+files = glob.glob(pattern, recursive=True)
 
 for file_path in files:
     if file_path.endswith('.py') and not os.path.basename(file_path).startswith('_'):

evalscope/benchmarks/aigc/t2i/base.py

@@ -0,0 +1,56 @@
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import DataAdapter
+from evalscope.metrics import mean, metric_registry
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class T2IBaseAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+
+        super().__init__(**kwargs)
+
+        logger.info(f'Initializing metrics: {self.metric_list}')
+        self.metrics = {m: metric_registry.get(m).object() for m in self.metric_list}
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        # dummy prompt for general t2i
+        return self.gen_prompt_data(prompt=input_d.get('prompt', ''), id=input_d.get('id', 0))
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # dummy gold answer for general t2i
+        return input_d.get('prompt', '')
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+        # dummy parse pred result for general t2i
+        return result or raw_input_d.get('image_path', '')
+
+    def match(self, gold: str, pred: str) -> dict:
+        # dummy match for general t2i
+        # pred is the image path, gold is the prompt
+        res = {}
+        for metric_name, metric_func in self.metrics.items():
+            score = metric_func(images=[pred], texts=[gold])[0][0]
+            if isinstance(score, dict):
+                for k, v in score.items():
+                    res[f'{metric_name}_{k}'] = v.cpu().item()
+            else:
+                res[metric_name] = score.cpu().item()  # Updated to use score.cpu().item()
+        return res
+
+    def compute_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
+        """
+        compute weighted mean of the bleu score of all samples
+
+        Args:
+            review_res_list: [score1, score2, ...]
+
+        Returns:
+            avg_res: List[dict]
+
+        """
+        items = super().compute_dict_metric(review_res_list, **kwargs)
+        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in items.items()]

evalscope/benchmarks/aigc/t2i/evalmuse_adapter.py

@@ -0,0 +1,77 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import Benchmark
+from evalscope.constants import OutputType
+from evalscope.metrics import mean
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+from .base import T2IBaseAdapter
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='evalmuse',
+    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+    model_adapter=OutputType.IMAGE_GENERATION,
+    output_types=[OutputType.IMAGE_GENERATION],
+    subset_list=['EvalMuse'],
+    metric_list=['FGA_BLIP2Score'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class EvalMuseAdapter(T2IBaseAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs) -> dict:
+        if os.path.isfile(self.dataset_id):
+            data_list = jsonl_to_list(self.dataset_id)
+            data_dict = {self.subset_list[0]: {'test': data_list}}
+            return data_dict
+        else:
+            return super().load(**kwargs)
+
+    def get_gold_answer(self, input_d: dict) -> dict:
+        # return prompt and elements dict
+        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
+
+    def match(self, gold: dict, pred: str) -> dict:
+        # dummy match for general t2i
+        # pred is the image path, gold is the prompt
+        res = {}
+        for metric_name, metric_func in self.metrics.items():
+            if metric_name == 'FGA_BLIP2Score':
+                # For FGA_BLIP2Score, we need to pass the dictionary
+                score = metric_func(images=[pred], texts=[gold])[0][0]
+            else:
+                score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
+            if isinstance(score, dict):
+                for k, v in score.items():
+                    res[f'{metric_name}:{k}'] = v.cpu().item()
+            else:
+                res[metric_name] = score.cpu().item()
+        return res
+
+    def compute_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
+        """
+        compute weighted mean of the bleu score of all samples
+        """
+        items = super().compute_dict_metric(review_res_list, **kwargs)
+        # add statistics for each metric
+        new_items = defaultdict(list)
+        for metric_name, value_list in items.items():
+            if 'FGA_BLIP2Score' in metric_name and '(' in metric_name:  # FGA_BLIP2Score element score
+                metrics_prefix = metric_name.split(':')[0]
+                category = metric_name.rpartition('(')[-1].split(')')[0]
+                new_items[f'{metrics_prefix}:{category}'].extend(value_list)
+            else:
+                new_items[metric_name].extend(value_list)
+
+        # calculate mean for each metric
+        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in new_items.items()]

evalscope/benchmarks/aigc/t2i/genai_bench_adapter.py

@@ -0,0 +1,58 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import Benchmark
+from evalscope.constants import OutputType
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+from .base import T2IBaseAdapter
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='genai_bench',
+    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+    model_adapter=OutputType.IMAGE_GENERATION,
+    output_types=[OutputType.IMAGE_GENERATION],
+    subset_list=['GenAI-Bench-1600'],
+    metric_list=['VQAScore'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class GenAIBenchAdapter(T2IBaseAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs) -> dict:
+        if os.path.isfile(self.dataset_id):
+            data_list = jsonl_to_list(self.dataset_id)
+            data_dict = {self.subset_list[0]: {'test': data_list}}
+            return data_dict
+        else:
+            return super().load(**kwargs)
+
+    def get_gold_answer(self, input_d: dict) -> dict:
+        # return prompt and elements dict
+        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
+
+    def match(self, gold: dict, pred: str) -> dict:
+        # dummy match for general t2i
+        # pred is the image path, gold is the prompt
+        res = {}
+        for metric_name, metric_func in self.metrics.items():
+            score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
+
+            res[metric_name] = score.cpu().item()
+
+            # fine-granular metrics
+            if gold['tags'].get('advanced'):
+                res[f'{metric_name}_advanced'] = score.cpu().item()
+            else:
+                res[f'{metric_name}_basic'] = score.cpu().item()
+
+        return res

evalscope/benchmarks/aigc/t2i/general_t2i_adapter.py

@@ -0,0 +1,58 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import Benchmark
+from evalscope.constants import OutputType
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+from .base import T2IBaseAdapter
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='general_t2i',
+    dataset_id='general_t2i',
+    model_adapter=OutputType.IMAGE_GENERATION,
+    output_types=[OutputType.IMAGE_GENERATION],
+    subset_list=['default'],
+    metric_list=['PickScore'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class GeneralT2IAdapter(T2IBaseAdapter):
+
+    def __init__(self, **kwargs):
+
+        super().__init__(**kwargs)
+
+    def load(self, dataset_name_or_path: str = None, subset_list: list = None, **kwargs) -> dict:
+        dataset_name_or_path = dataset_name_or_path or self.dataset_id
+        subset_list = subset_list or self.subset_list
+
+        data_file_dict = defaultdict(str)
+        data_item_dict = defaultdict(list)
+
+        # get data file path and subset name
+        if os.path.isdir(dataset_name_or_path):
+            for subset_name in subset_list:
+                data_file_dict[subset_name] = os.path.join(dataset_name_or_path, f'{subset_name}.jsonl')
+        elif os.path.isfile(dataset_name_or_path):
+            cur_subset_name = os.path.splitext(os.path.basename(dataset_name_or_path))[0]
+            data_file_dict[cur_subset_name] = dataset_name_or_path
+        else:
+            raise ValueError(f'Invalid dataset path: {dataset_name_or_path}')
+
+        # load data from local disk
+        try:
+            for subset_name, file_path in data_file_dict.items():
+                data_item_dict[subset_name] = jsonl_to_list(file_path)
+        except Exception as e:
+            raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
+
+        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
+
+        return data_dict

evalscope/benchmarks/aigc/t2i/hpdv2_adapter.py

@@ -0,0 +1,57 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import Benchmark
+from evalscope.constants import OutputType
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+from .base import T2IBaseAdapter
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='hpdv2',
+    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+    model_adapter=OutputType.IMAGE_GENERATION,
+    output_types=[OutputType.IMAGE_GENERATION],
+    subset_list=['HPDv2'],
+    metric_list=['HPSv2.1Score'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class HPDv2Adapter(T2IBaseAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs) -> dict:
+        if os.path.isfile(self.dataset_id):
+            data_list = jsonl_to_list(self.dataset_id)
+            data_dict = {self.subset_list[0]: {'test': data_list}}
+            return data_dict
+        else:
+            return super().load(**kwargs)
+
+    def get_gold_answer(self, input_d: dict) -> dict:
+        # return prompt and elements dict
+        return {'prompt': input_d.get('prompt'), 'tags': input_d.get('tags', {})}
+
+    def match(self, gold: dict, pred: str) -> dict:
+        # dummy match for general t2i
+        # pred is the image path, gold is the prompt
+        res = {}
+        for metric_name, metric_func in self.metrics.items():
+            score = metric_func(images=[pred], texts=[gold['prompt']])[0][0]
+
+            res[metric_name] = score.cpu().item()
+
+            # fine-granular metrics
+            category = gold['tags'].get('category')
+            if category:
+                res[f'{metric_name}_{category}'] = score.cpu().item()
+
+        return res

evalscope/benchmarks/aigc/t2i/tifa_adapter.py

@@ -0,0 +1,37 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os.path
+from collections import defaultdict
+from typing import List, Optional, Union
+
+from evalscope.benchmarks import Benchmark
+from evalscope.constants import OutputType
+from evalscope.utils.io_utils import jsonl_to_list
+from evalscope.utils.logger import get_logger
+from .base import T2IBaseAdapter
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='tifa160',
+    dataset_id='AI-ModelScope/T2V-Eval-Prompts',
+    model_adapter=OutputType.IMAGE_GENERATION,
+    output_types=[OutputType.IMAGE_GENERATION],
+    subset_list=['TIFA-160'],
+    metric_list=['PickScore'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+)
+class TIFA_Adapter(T2IBaseAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs) -> dict:
+        if os.path.isfile(self.dataset_id):
+            data_list = jsonl_to_list(self.dataset_id)
+            data_dict = {self.subset_list[0]: {'test': data_list}}
+            return data_dict
+        else:
+            return super().load(**kwargs)

evalscope/benchmarks/aime/aime24_adapter.py

@@ -1,6 +1,6 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import OutputType
-from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
+from evalscope.metrics import extract_answer, math_equal, strip_answer_string
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa

evalscope/benchmarks/aime/aime25_adapter.py

@@ -1,6 +1,6 @@
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import OutputType
-from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
+from evalscope.metrics import extract_answer, math_equal, strip_answer_string
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -11,12 +11,12 @@ logger = get_logger()
 @Benchmark.register(
     name='aime25',
     pretty_name='AIME-2025',
-    dataset_id='TIGER-Lab/AIME25',
-    subset_list=['default'],
+    dataset_id='opencompass/AIME2025',
+    subset_list=['AIME2025-I', 'AIME2025-II'],
     metric_list=['AveragePass@1'],
     few_shot_num=0,
     train_split=None,
-    eval_split='train',  # Only train set is available
+    eval_split='test',  # Only train set is available
     prompt_template='{query}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
 )
 class AIME25Adapter(DataAdapter):

evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py

@@ -3,8 +3,7 @@ from collections import defaultdict
 from typing import Any, List
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import Metric, mean, metric_registry
-from evalscope.metrics.llm_judge import LLMJudge
+from evalscope.metrics import LLMJudge, Metric, mean, metric_registry
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa

evalscope/benchmarks/arc/arc_adapter.py

@@ -18,7 +18,7 @@ logger = get_logger()
     name='arc',
     pretty_name='ARC',
     dataset_id='modelscope/ai2_arc',
-    model_adapter=OutputType.MULTIPLE_CHOICE,
+    model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['ARC-Easy', 'ARC-Challenge'],
     metric_list=['AverageAccuracy'],

evalscope/benchmarks/arena_hard/arena_hard_adapter.py

@@ -3,9 +3,7 @@ from collections import defaultdict
 from typing import Any, List
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import AnswerKeys
-from evalscope.metrics import Metric, mean, metric_registry
-from evalscope.metrics.llm_judge import LLMJudge
+from evalscope.metrics import LLMJudge, Metric, mean, metric_registry
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -4,7 +4,7 @@ import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType, OutputType
-from evalscope.metrics.metrics import exact_match
+from evalscope.metrics import exact_match
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -127,7 +127,7 @@ SUBJECT_MAPPING = {
     name='ceval',
     pretty_name='C-Eval',
     dataset_id='modelscope/ceval-exam',
-    model_adapter=OutputType.MULTIPLE_CHOICE,
+    model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],

evalscope/benchmarks/chinese_simple_qa/csimple_qa_adapter.py

@@ -1,10 +1,8 @@
 import re
-from collections import defaultdict
 from typing import Any, List
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import Metric, mean, metric_registry
-from evalscope.metrics.llm_judge import LLMJudge
+from evalscope.metrics import LLMJudge, Metric, mean, metric_registry
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -104,7 +104,7 @@ SUBJECT_MAPPING = {
     name='cmmlu',
     pretty_name='C-MMLU',
     dataset_id='modelscope/cmmlu',
-    model_adapter=OutputType.MULTIPLE_CHOICE,
+    model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=SUBSET_LIST,
     metric_list=['AverageAccuracy'],

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -6,8 +6,7 @@ import os
 from collections import defaultdict
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import AnswerKeys
-from evalscope.metrics.math_parser import extract_answer, math_equal, strip_answer_string
+from evalscope.metrics import extract_answer, math_equal, strip_answer_string
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa

evalscope/benchmarks/data_adapter.py

@@ -3,12 +3,11 @@ import os.path
 import random
 from abc import ABC, abstractmethod
 from collections import defaultdict
-from typing import Any, List, Optional, Union
+from typing import Any, Dict, List, Optional, Union
 
 from evalscope.benchmarks.utils import PromptData, preprocess_decorator
 from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, AnswerKeys, EvalType, HubType
-from evalscope.metrics.llm_judge import LLMJudge
-from evalscope.metrics.named_metrics import metric_registry
+from evalscope.metrics import LLMJudge, metric_registry
 from evalscope.report import Report, ReportGenerator
 from evalscope.utils.logger import get_logger
 
@@ -24,6 +23,7 @@ class DataAdapter(ABC):
                  subset_list: list,
                  metric_list: List[str],
                  llm_as_a_judge: bool = False,
+                 output_types: Optional[List[str]] = None,
                  few_shot_num: Optional[int] = 0,
                  train_split: Optional[str] = None,
                  eval_split: Optional[str] = None,
@@ -63,6 +63,7 @@ class DataAdapter(ABC):
         self.query_template = query_template
         self.pretty_name = pretty_name
         self.config_kwargs = kwargs
+        self.output_types = output_types or [model_adapter]
         self.llm_as_a_judge = llm_as_a_judge
         self.category_map = kwargs.get('category_map', {})
         self.choices = kwargs.get('choices', None)
@@ -190,7 +191,7 @@ class DataAdapter(ABC):
         if self.few_shot_num and self.few_shot_num < 0:
             raise ValueError(f'Invalid shot_num: {self.few_shot_num} for few-shot evaluation.')
 
-        logger.info(f'Use default settings: '
+        logger.info(f'Use settings: '
                     f'> few_shot_num: {self.few_shot_num}, '
                     f'> few_shot_split: {self.train_split}, '
                     f'> target_eval_split: {self.eval_split}')
@@ -245,7 +246,8 @@ class DataAdapter(ABC):
             res_list.append({'metric_name': metric_name, 'score': metric_func(review_res), 'num': len(review_res)})
         return res_list
 
-    def compute_dict_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
+    def compute_dict_metric(self, review_res_list: Union[List[dict], List[List[dict]]],
+                            **kwargs) -> Dict[str, List[float]]:
         """
         compute weighted mean of the bleu score of all samples
 
@@ -253,7 +255,7 @@ class DataAdapter(ABC):
             review_res_list: [score1, score2, ...]
 
         Returns:
-            avg_res: List[dict]
+            avg_res: Dict[str, List[float]]
 
         """
         if isinstance(review_res_list[0], list):
@@ -318,11 +320,16 @@ class DataAdapter(ABC):
                         prompt: str,
                         system_prompt: Optional[str] = None,
                         choices: Optional[List[str]] = None,
+                        index: Optional[Union[int, str]] = None,
+                        id: Optional[Union[int, str]] = None,
                         **kwargs) -> dict:
-        if not isinstance(prompt, list):
-            prompt = [prompt]
+        data = [prompt] if not isinstance(prompt, list) else prompt
         prompt_data = PromptData(
-            data=prompt, multi_choices=choices or self.choices, system_prompt=system_prompt or self.system_prompt)
+            data=data,
+            multi_choices=choices or self.choices,
+            system_prompt=system_prompt or self.system_prompt,
+            index=index or 0,
+            id=id)
         return prompt_data.to_dict()
 
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -48,14 +48,16 @@ class DataCollectionAdapter(DataAdapter):
             if len(dataset) == 0:
                 raise ValueError(f'Local dataset is empty: {dataset_name_or_path}')
         else:
-            from modelscope.msdatasets import MsDataset
+            from modelscope import dataset_snapshot_download
 
             # Load dataset from remote
             logger.info(f'Loading dataset from {datasets_hub}: > dataset_name: {dataset_name_or_path}')
 
-            dataset = MsDataset.load(dataset_name=dataset_name_or_path, cache_dir=work_dir, hub=datasets_hub, **kwargs)
-
-            dataset = dataset[self.eval_split].to_list()
+            dataset_path = dataset_snapshot_download(
+                dataset_name_or_path, cache_dir=work_dir, allow_file_pattern='*.jsonl')
+            # find the jsonl file
+            dataset_files = [os.path.join(dataset_path, f) for f in os.listdir(dataset_path) if f.endswith('.jsonl')]
+            dataset = jsonl_to_list(dataset_files[0])
 
         return dataset
 

evalscope/benchmarks/general_mcq/general_mcq_adapter.py

@@ -4,7 +4,7 @@ import os
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType, OutputType
-from evalscope.metrics.metrics import exact_match
+from evalscope.metrics import exact_match
 from evalscope.utils import ResponseParser
 from evalscope.utils.logger import get_logger
 
@@ -17,7 +17,7 @@ logger = get_logger()
     name='general_mcq',
     pretty_name='General MCQ',
     dataset_id='general_mcq',
-    model_adapter=OutputType.MULTIPLE_CHOICE,
+    model_adapter=OutputType.GENERATION,
     output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
     subset_list=['default'],
     metric_list=['AverageAccuracy'],

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -33,7 +33,7 @@ class GeneralQAAdapter(DataAdapter):
         subset_list = subset_list or self.subset_list
 
         data_file_dict = defaultdict(str)
-        data_list = []
+        data_item_dict = defaultdict(list)
 
         # get data file path and subset name
         if os.path.isdir(dataset_name_or_path):
@@ -48,11 +48,11 @@ class GeneralQAAdapter(DataAdapter):
         # load data from local disk
         try:
             for subset_name, file_path in data_file_dict.items():
-                data_list.extend(jsonl_to_list(file_path))
+                data_item_dict[subset_name] = jsonl_to_list(file_path)
         except Exception as e:
             raise ValueError(f'Failed to load data from {self.dataset_id}, got error: {e}')
 
-        data_dict = {subset_name: {'test': data_list} for subset_name in data_file_dict.keys()}
+        data_dict = {subset_name: {'test': data_item_dict[subset_name]} for subset_name in data_file_dict.keys()}
 
         return data_dict