evalscope 0.16.3__py3-none-any.whl → 0.17.1__py3-none-any.whl

evalscope/backend/vlm_eval_kit/backend_manager.py

@@ -5,7 +5,8 @@ from functools import partial
 from typing import Optional, Union
 
 from evalscope.backend.base import BackendManager
-from evalscope.utils import get_valid_list, is_module_installed
+from evalscope.utils.import_utils import is_module_installed
+from evalscope.utils.io_utils import get_valid_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -68,6 +69,8 @@ class VLMEvalKitBackendManager(BackendManager):
                     del remain_cfg['type']  # remove not used args
 
                     norm_model_type = os.path.basename(model_type).replace(':', '-').replace('.', '_')
+                    model_cfg['type'] = norm_model_type
+
                     self.valid_models.update({norm_model_type: partial(model_class, model=model_type, **remain_cfg)})
                     new_model_names.append(norm_model_type)
                 else:

evalscope/benchmarks/__init__.py

@@ -2,6 +2,7 @@
 import glob
 import importlib
 import os
+import time
 
 from evalscope.benchmarks.benchmark import Benchmark, BenchmarkMeta
 from evalscope.benchmarks.data_adapter import DataAdapter
@@ -13,11 +14,24 @@ logger = get_logger()
 pattern = os.path.join(os.path.dirname(__file__), '*', '**', '*_adapter.py')
 files = glob.glob(pattern, recursive=True)
 
+import_times = []
+
 for file_path in files:
     if file_path.endswith('.py') and not os.path.basename(file_path).startswith('_'):
         # Convert file path to a module path
         relative_path = os.path.relpath(file_path, os.path.dirname(__file__))
         module_path = relative_path[:-3].replace(os.path.sep, '.')  # strip '.py' and convert to module path
         full_path = f'evalscope.benchmarks.{module_path}'
+
+        start_time = time.perf_counter()
         importlib.import_module(full_path)
-        # print(f'Importing {full_path}')
+        end_time = time.perf_counter()
+
+        import_times.append((full_path, end_time - start_time))
+
+# Sort by import time in descending order
+import_times.sort(key=lambda x: x[1], reverse=True)
+
+# Log the sorted import times
+for module, duration in import_times:
+    logger.debug(f'Module {module} imported in {duration:.6f} seconds')

evalscope/benchmarks/aime/aime24_adapter.py

@@ -48,4 +48,5 @@ class AIME24Adapter(DataAdapter):
         return result
 
     def match(self, gold: str, pred: str) -> float:
-        return math_equal(pred, gold)
+        res = math_equal(pred, gold)
+        return 1.0 if res else 0.0

evalscope/benchmarks/aime/aime25_adapter.py

@@ -48,4 +48,5 @@ class AIME25Adapter(DataAdapter):
         return result
 
     def match(self, gold: str, pred: str) -> float:
-        return math_equal(pred, gold)
+        res = math_equal(pred, gold)
+        return 1.0 if res else 0.0

evalscope/benchmarks/alpaca_eval/alpaca_eval_adapter.py

@@ -47,7 +47,7 @@ Evaluate the models based on the quality and relevance of their outputs, and sel
 @Benchmark.register(
     name='alpaca_eval',
     pretty_name='AlpacaEval2.0',
-    tags=['Instruction-Following', 'Reasoning'],
+    tags=['Instruction-Following', 'Arena'],
     description='Alpaca Eval 2.0 is an enhanced framework for evaluating instruction-following language models, '
     'featuring an improved auto-annotator, updated baselines, and continuous preference calculation to '
     'provide more accurate and cost-effective model assessments. '

evalscope/benchmarks/arc/arc_adapter.py

@@ -6,7 +6,7 @@ import os
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.utils import ResponseParser
+from evalscope.metrics.completion_parsers import ResponseParser
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa

evalscope/benchmarks/arena_hard/arena_hard_adapter.py

@@ -17,7 +17,7 @@ GRADER_TEMPLATE = "<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's A
 @Benchmark.register(
     name='arena_hard',
     pretty_name='ArenaHard',
-    tags=['Instruction-Following', 'Reasoning'],
+    tags=['Instruction-Following', 'Arena'],
     description=
     'ArenaHard is a benchmark designed to evaluate the performance of large language models in a competitive setting, '
     'where models are pitted against each other in a series of tasks to determine their relative strengths and weaknesses. '

evalscope/benchmarks/arena_hard/utils.py

@@ -127,18 +127,6 @@ def get_bootstrap_result(battles, func_compute_elo, num_round):
     return df[df.median().sort_values(ascending=False).index]
 
 
-def preety_print_two_ratings(ratings_1, ratings_2, column_names):
-    df = (
-        pd.DataFrame(
-            [[n, ratings_1[n], ratings_2[n]] for n in ratings_1.keys()],
-            columns=['Model', column_names[0], column_names[1]],
-        ).sort_values(column_names[0], ascending=False).reset_index(drop=True))
-    df[column_names[0]] = (df[column_names[0]] + 0.5).astype(int)
-    df[column_names[1]] = (df[column_names[1]] + 0.5).astype(int)
-    df.index = df.index + 1
-    return df
-
-
 def predict_win_rate(elo_ratings, SCALE=400, BASE=10, INIT_RATING=1000):
     names = sorted(list(elo_ratings.keys()))
     wins = defaultdict(lambda: defaultdict(lambda: 0))

evalscope/benchmarks/bfcl/bfcl_adapter.py

@@ -35,7 +35,7 @@ SUBJECT_MAPPING = {
 @Benchmark.register(
     name='bfcl_v3',
     pretty_name='BFCL-v3',
-    tags=['Agent'],
+    tags=['Agent', 'Function Calling'],
     description=
     'Berkeley Function Calling Leaderboard (BFCL), the **first comprehensive and executable function call evaluation** '
     'dedicated to assessing Large Language Models\' (LLMs) ability to invoke functions. Unlike previous evaluations, '

evalscope/benchmarks/ceval/ceval_adapter.py

@@ -1,11 +1,13 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 import csv
 import os
+from collections import defaultdict
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.utils import ResponseParser
+from evalscope.metrics.completion_parsers import ResponseParser
+from evalscope.utils.io_utils import csv_to_list
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -154,7 +156,7 @@ class CEVALAdapter(DataAdapter):
         self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
+        data_dict = defaultdict(dict)
         for subset_name in subset_list:
             for split_name in [self.train_split, self.eval_split]:
                 if os.path.exists(dataset_name_or_path):
@@ -162,20 +164,7 @@ class CEVALAdapter(DataAdapter):
                 else:
                     file_path = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}_{split_name}.csv')
                 if os.path.exists(file_path):
-                    with open(file_path, encoding='utf-8') as f:
-                        rows = []
-                        reader = csv.reader(f)
-                        header = next(reader)
-                        for row in reader:
-                            item = dict(zip(header, row))
-                            item.setdefault('explanation', '')
-                            item.setdefault('answer', '')
-                            rows.append(item)
-
-                        if subset_name in data_dict:
-                            data_dict[subset_name].update({split_name: rows})
-                        else:
-                            data_dict[subset_name] = {split_name: rows}
+                    data_dict[subset_name][split_name] = csv_to_list(file_path)
 
         return data_dict
 

evalscope/benchmarks/cmmlu/cmmlu_adapter.py

@@ -2,11 +2,13 @@
 
 import csv
 import os
+from collections import defaultdict
 
 from evalscope.benchmarks import Benchmark, DataAdapter
 from evalscope.constants import EvalType, OutputType
 from evalscope.metrics import exact_match
-from evalscope.utils import ResponseParser
+from evalscope.metrics.completion_parsers import ResponseParser
+from evalscope.utils.io_utils import csv_to_list
 from evalscope.utils.logger import get_logger
 
 # flake8: noqa
@@ -126,29 +128,15 @@ class CMMLUAdapter(DataAdapter):
         self.choices = ['A', 'B', 'C', 'D']
 
     def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
+        data_dict = defaultdict(dict)
         for subset_name in subset_list:
-            data_dict[subset_name] = {}
             for split_name in [self.train_split, self.eval_split]:
-                file_path = os.path.join(work_dir, dataset_name_or_path, split_name, f'{subset_name}.csv')
+                if os.path.exists(dataset_name_or_path):
+                    file_path = os.path.join(dataset_name_or_path, split_name, f'{subset_name}.csv')
+                else:
+                    file_path = os.path.join(work_dir, dataset_name_or_path, split_name, f'{subset_name}.csv')
                 if os.path.exists(file_path):
-                    with open(file_path, encoding='utf-8') as f:
-                        rows = []
-                        reader = csv.reader(f)
-                        for row in reader:
-                            if len(row) != 7:
-                                logger.error(f'Mismatch len of row: {row}, len of row should be 6. Skip this row.')
-                                continue
-                            rows.append({
-                                'Question': row[1],
-                                'A': row[2],
-                                'B': row[3],
-                                'C': row[4],
-                                'D': row[5],
-                                'Answer': row[6],
-                            })
-
-                        data_dict[subset_name].update({split_name: rows})
+                    data_dict[subset_name][split_name] = csv_to_list(file_path)
 
         return data_dict
 

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -105,7 +105,8 @@ class CompetitionMathAdapter(DataAdapter):
         return result
 
     def match(self, gold: str, pred: str) -> float:
-        return math_equal(pred, gold)
+        res = math_equal(pred, gold)
+        return 1.0 if res else 0.0
 
     @classmethod
     def _generate_prompt(cls, input_d: dict, use_fewshot: bool = True) -> str:

evalscope/benchmarks/data_adapter.py

@@ -168,7 +168,12 @@ class DataAdapter(ABC):
         If you want to support local dataset, please rewrite this method in xxx_data_adapter.
         Use modelscope.msdatasets.MsDataset.load to load the dataset from local by default.
         """
-        return self.load_from_hub(dataset_name_or_path, subset_list, work_dir, **kwargs)
+        # remove dataset_infos.json file if exists, since MsDataset will occur an error if it exists.
+        dataset_infos_path = os.path.join(dataset_name_or_path, 'dataset_infos.json')
+        if os.path.exists(dataset_infos_path):
+            logger.info(f'Removing dataset_infos.json file at {dataset_infos_path} to avoid MsDataset errors.')
+            os.remove(dataset_infos_path)
+        return self.load_from_hub(dataset_name_or_path, subset_list, None, **kwargs)
 
     def load_with_snapshot(self,
                            file_structure: Dict[str, List[str]],
@@ -382,7 +387,7 @@ class DataAdapter(ABC):
         pass
 
     def gen_prompt_data(self,
-                        prompt: str,
+                        prompt: str = '',
                         system_prompt: Optional[str] = None,
                         choices: Optional[List[str]] = None,
                         index: Optional[Union[int, str]] = None,
@@ -413,7 +418,8 @@ class DataAdapter(ABC):
             system_prompt=system_prompt or self.system_prompt,
             index=index or 0,
             id=id,
-            messages=messages)
+            messages=messages,
+            extra_data=kwargs.get('extra_data', None))
         return prompt_data.to_dict()
 
     def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
@@ -449,7 +455,6 @@ class DataAdapter(ABC):
         """
         raise NotImplementedError
 
-    @abstractmethod
     def parse_pred_result(self, result: Any, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> Any:
         """
         Parse the predicted result and extract proper answer.
@@ -462,9 +467,22 @@ class DataAdapter(ABC):
         Returns:
             The parsed answer. Depending on the dataset. Usually a string for chat.
         """
-        raise NotImplementedError
+        return result
+
+    def llm_parse_pred_result(self, result: Any, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> Any:
+        """
+        Parse the predicted result using LLM.
+
+        Args:
+            result (Any): The predicted answer from the model.
+            raw_input_d (dict): The raw input data.
+            eval_type (str): The evaluation type, default is 'checkpoint'.
+
+        Returns:
+            The parsed answer. Usually a string for chat.
+        """
+        return result
 
-    @abstractmethod
     def match(self, gold: Any, pred: Any) -> Any:
         """
         Match the gold answer and the predicted answer.
@@ -478,7 +496,7 @@ class DataAdapter(ABC):
         Returns:
             The match result. Usually a score (float) for chat/multiple-choice-questions.
         """
-        raise NotImplementedError
+        return 1.0 if gold == pred else 0.0
 
     def llm_match(self, gold: Any, pred: Any, judge: Optional[LLMJudge] = None, **kwargs) -> float:
         """
@@ -504,5 +522,7 @@ class DataAdapter(ABC):
 
         # Request judge and obtain score
         prompt = judge.build_prompt(pred, gold, question)
-        score = judge(prompt)
-        return judge.get_score(score)
+        judge_response = judge(prompt)
+        score = judge.get_score(judge_response)
+
+        return score