evalscope 0.8.2__py3-none-any.whl → 0.10.0__py3-none-any.whl

evalscope/report/utils.py

@@ -0,0 +1,133 @@
+import json
+import pandas as pd
+from collections import defaultdict
+from dataclasses import asdict, dataclass, field
+from typing import Any, Dict, List
+
+from evalscope.metrics import macro_mean, micro_mean
+from evalscope.utils import normalize_score
+
+
+@dataclass
+class Subset:
+    name: str = 'default_subset'
+    score: float = 0.0
+    num: int = 0
+
+    def __post_init__(self):
+        self.score = normalize_score(self.score)
+
+
+@dataclass
+class Category:
+    name: tuple[str] = field(default_factory=tuple)
+    num: int = 0
+    score: float = 0.0
+    macro_score: float = 0.0
+    subsets: List[Subset] = field(default_factory=list)
+
+    def __post_init__(self):
+        if isinstance(self.name, str):
+            # ensure name is tuple format
+            self.name = (self.name, )
+        self.num = sum(subset.num for subset in self.subsets)
+        self.score = normalize_score(micro_mean(self.subsets))
+        self.macro_score = normalize_score(macro_mean(self.subsets))
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        subsets = [Subset(**subset) for subset in data.get('subsets', [])]
+        return cls(name=data['name'], subsets=subsets)
+
+
+@dataclass
+class Metric:
+    name: str = 'default_metric'
+    num: int = 0
+    score: float = 0.0
+    macro_score: float = 0.0
+    categories: List[Category] = field(default_factory=list)
+
+    def __post_init__(self):
+        self.num = sum(category.num for category in self.categories)
+        self.score = normalize_score(micro_mean(self.categories))
+        self.macro_score = normalize_score(macro_mean(self.categories))
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        categories = [Category.from_dict(category) for category in data.get('categories', [])]
+        return cls(name=data['name'], categories=categories)
+
+
+class ReportKey:
+    model_name = 'Model'
+    dataset_name = 'Dataset'
+    metric_name = 'Metric'
+    category_name = 'Category'
+    category_prefix = 'Cat.'
+    subset_name = 'Subset'
+    num = 'Num'
+    score = 'Score'
+
+
+@dataclass
+class Report:
+    name: str = 'default_report'
+    dataset_name: str = 'default_dataset'
+    model_name: str = 'default_model'
+    score: float = 0.0
+    metrics: List[Metric] = field(default_factory=list)
+
+    def __post_init__(self):
+        self.score = self.metrics[0].score  # NOTE: only use the first metric by default
+
+    def to_dict(self) -> Dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict):
+        metrics = [Metric.from_dict(metric) for metric in data.get('metrics', [])]
+        return cls(
+            name=data['name'],
+            score=data['score'],
+            metrics=metrics,
+            dataset_name=data['dataset_name'],
+            model_name=data['model_name'])
+
+    @classmethod
+    def from_json(cls, json_file: str):
+        with open(json_file, 'r') as f:
+            data = json.load(f)
+        return cls.from_dict(data)
+
+    def to_dataframe(self, flatten_metrics: bool = True, flatten_categories: bool = True):
+        table = defaultdict(list)
+        for metric in self.metrics:
+            for category in metric.categories:
+                for subset in category.subsets:
+                    table[ReportKey.model_name].append(self.model_name)
+                    table[ReportKey.dataset_name].append(self.dataset_name)
+                    table[ReportKey.metric_name].append(metric.name)
+                    table[ReportKey.category_name].append(category.name)
+                    table[ReportKey.subset_name].append(subset.name)
+                    table[ReportKey.num].append(subset.num)
+                    table[ReportKey.score].append(subset.score)  # TODO: convert to percentage
+            # NOTE: only flatten metrics if needed, use the first metric by default
+            if not flatten_metrics:
+                break
+        df = pd.DataFrame.from_dict(table, orient='columns')
+        if flatten_categories:
+            df = self._flatten_categories(df)
+        return df
+
+    def _flatten_categories(self, df: pd.DataFrame):
+        # expand categories to multiple rows
+        df_categories = df.copy()
+        # multi-level aggregation for categories
+        max_depth = df_categories[ReportKey.category_name].apply(len).max()
+        for level in range(max_depth):
+            df_categories[f'{ReportKey.category_prefix}{level}'] = df_categories[ReportKey.category_name].apply(
+                lambda x: x[level] if len(x) > level else None)
+
+        df_categories.drop(columns=[ReportKey.category_name], inplace=True)
+        return df_categories

evalscope/run.py

@@ -2,26 +2,21 @@
 """
 Run evaluation for LLMs.
 """
-import logging
 import os.path
-import torch
 from argparse import Namespace
 from datetime import datetime
-from typing import List, Optional, Union
+from typing import TYPE_CHECKING, List, Optional, Union
 
-from evalscope.arguments import parse_args
 from evalscope.config import TaskConfig, parse_task_config
-from evalscope.constants import DEFAULT_MODEL_REVISION, DEFAULT_WORK_DIR, EvalBackend, EvalType
-from evalscope.evaluator import Evaluator
-from evalscope.models.custom import CustomModel
-from evalscope.utils import import_module_util, seed_everything
-from evalscope.utils.io_utils import OutputsStructure, are_paths_same
+from evalscope.constants import DataCollection, EvalBackend
+from evalscope.utils import seed_everything
+from evalscope.utils.io_utils import OutputsStructure
 from evalscope.utils.logger import configure_logging, get_logger
 
-logger = get_logger()
+if TYPE_CHECKING:
+    from evalscope.models import LocalModel
 
-BENCHMARK_PATH_PREFIX = 'evalscope.benchmarks.'
-MEMBERS_TO_IMPORT = ['DATASET_ID', 'SUBSET_LIST', 'DataAdapterClass', 'ModelAdapterClass']
+logger = get_logger()
 
 
 def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig], Namespace]) -> Union[dict, List[dict]]:
@@ -38,15 +33,13 @@ def run_task(task_cfg: Union[str, dict, TaskConfig, List[TaskConfig], Namespace]
 
 def run_single_task(task_cfg: TaskConfig, run_time: str) -> dict:
     """Run a single evaluation task."""
-    seed_everything(task_cfg.seed)
+    if task_cfg.seed is not None:
+        seed_everything(task_cfg.seed)
     outputs = setup_work_directory(task_cfg, run_time)
     configure_logging(task_cfg.debug, os.path.join(outputs.logs_dir, 'eval_log.log'))
 
-    task_cfg.dump_yaml(outputs.configs_dir)
-    logger.info(task_cfg)
-
     if task_cfg.eval_backend != EvalBackend.NATIVE:
-        return run_non_native_backend(task_cfg)
+        return run_non_native_backend(task_cfg, outputs)
     else:
         return evaluate_model(task_cfg, outputs)
 
@@ -56,8 +49,8 @@ def setup_work_directory(task_cfg: TaskConfig, run_time: str):
     if task_cfg.use_cache:
         task_cfg.work_dir = task_cfg.use_cache
         logger.info(f'Set resume from {task_cfg.work_dir}')
-    elif are_paths_same(task_cfg.work_dir, DEFAULT_WORK_DIR):
-        task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
+    # elif are_paths_same(task_cfg.work_dir, DEFAULT_WORK_DIR):
+    task_cfg.work_dir = os.path.join(task_cfg.work_dir, run_time)
 
     outputs = OutputsStructure(outputs_dir=task_cfg.work_dir)
 
@@ -68,7 +61,7 @@ def setup_work_directory(task_cfg: TaskConfig, run_time: str):
     return outputs
 
 
-def run_non_native_backend(task_cfg: TaskConfig) -> dict:
+def run_non_native_backend(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     """Run evaluation using a non-native backend."""
     eval_backend = task_cfg.eval_backend
     eval_config = task_cfg.eval_config
@@ -78,6 +71,10 @@ def run_non_native_backend(task_cfg: TaskConfig) -> dict:
 
     backend_manager_class = get_backend_manager_class(eval_backend)
     backend_manager = backend_manager_class(config=eval_config)
+
+    task_cfg.dump_yaml(outputs.configs_dir)
+    logger.info(task_cfg)
+
     backend_manager.run()
 
     return dict()
@@ -100,78 +97,57 @@ def get_backend_manager_class(eval_backend: EvalBackend):
 
 def evaluate_model(task_cfg: TaskConfig, outputs: OutputsStructure) -> dict:
     """Evaluate the model based on the provided task configuration."""
+    from evalscope.models import get_local_model
+
     # Initialize evaluator
     eval_results = {}
-
+    base_model = get_local_model(task_cfg)
+    evaluators = []
     for dataset_name in task_cfg.datasets:
-        evaluator = create_evaluator(task_cfg, dataset_name, outputs)
+        evaluator = create_evaluator(task_cfg, dataset_name, outputs, base_model)
+        evaluators.append(evaluator)
+
+    # dump task_cfg to outputs.configs_dir after creating evaluators
+    task_cfg.dump_yaml(outputs.configs_dir)
+    logger.info(task_cfg)
+
+    for evaluator in evaluators:
         res_dict = evaluator.eval(infer_cfg=task_cfg.generation_config, debug=task_cfg.debug, limit=task_cfg.limit)
         eval_results[dataset_name] = res_dict
 
     return eval_results
 
 
-def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsStructure):
+def create_evaluator(task_cfg: TaskConfig, dataset_name: str, outputs: OutputsStructure, base_model: 'LocalModel'):
     """Create an evaluator object for the specified dataset."""
-    imported_modules = import_module_util(BENCHMARK_PATH_PREFIX, dataset_name, MEMBERS_TO_IMPORT)
-    model_adapter = initialize_model_adapter(task_cfg, dataset_name, imported_modules)
-
-    dataset_config = task_cfg.dataset_args.get(dataset_name, {})
-    dataset_name_or_path = dataset_config.get('local_path') or imported_modules['DATASET_ID']
-    in_prompt_template = dataset_config.get('prompt_template', '')
-    few_shot_num = dataset_config.get('few_shot_num', None)
-    few_shot_random = dataset_config.get('few_shot_random', True)
-
-    data_adapter = imported_modules['DataAdapterClass'](
-        few_shot_num=few_shot_num,
-        few_shot_random=few_shot_random,
-        prompt_template=in_prompt_template,
-        outputs=outputs,
-    )
-    in_subset_list = dataset_config.get('subset_list', imported_modules['SUBSET_LIST'])
+    from evalscope.benchmarks import Benchmark, BenchmarkMeta
+    from evalscope.evaluator import Evaluator
+    from evalscope.models import initialize_model_adapter
+
+    if dataset_name == DataCollection.NAME:
+        # EvaluatorCollection is a collection of evaluators
+        from evalscope.collections import EvaluatorCollection
+        return EvaluatorCollection(task_cfg, outputs)
 
-    logger.info(f'Evaluating on subsets for {dataset_name}: {in_subset_list}\n')
+    benchmark: BenchmarkMeta = Benchmark.get(dataset_name)
+
+    data_adapter = benchmark.get_data_adapter(config=task_cfg.dataset_args.get(dataset_name, {}))
+    model_adapter = initialize_model_adapter(task_cfg, benchmark.model_adapter, base_model)
+
+    # update task_cfg.dataset_args
+    task_cfg.dataset_args[dataset_name] = benchmark.to_string_dict()
 
     return Evaluator(
-        dataset_name_or_path=dataset_name_or_path,
-        subset_list=in_subset_list,
+        dataset_name_or_path=benchmark.dataset_id,
         data_adapter=data_adapter,
         model_adapter=model_adapter,
-        use_cache=task_cfg.use_cache,
         outputs=outputs,
-        datasets_dir=task_cfg.dataset_dir,
-        datasets_hub=task_cfg.dataset_hub,
-        stage=task_cfg.stage,
-        eval_type=task_cfg.eval_type,
-        overall_task_cfg=task_cfg,
+        task_cfg=task_cfg,
     )
 
 
-def initialize_model_adapter(task_cfg: TaskConfig, dataset_name: str, imported_modules):
-    """Initialize the model adapter based on the task configuration."""
-    if task_cfg.dry_run:
-        from evalscope.models.dummy_chat_model import DummyChatModel
-        return DummyChatModel(model_cfg=dict())
-    elif task_cfg.eval_type == EvalType.CUSTOM:
-        if not isinstance(task_cfg.model, CustomModel):
-            raise ValueError(f'Expected evalscope.models.custom.CustomModel, but got {type(task_cfg.model)}.')
-        from evalscope.models.model_adapter import CustomModelAdapter
-        return CustomModelAdapter(custom_model=task_cfg.model)
-    else:
-        device_map = task_cfg.model_args.get('device_map', 'auto') if torch.cuda.is_available() else None
-        model_precision = task_cfg.model_args.get('precision', torch.float16)
-        if isinstance(model_precision, str) and model_precision != 'auto':
-            model_precision = eval(model_precision)
-        return imported_modules['ModelAdapterClass'](
-            model_id=task_cfg.model,
-            model_revision=task_cfg.model_args.get('revision', DEFAULT_MODEL_REVISION),
-            device_map=device_map,
-            torch_dtype=model_precision,
-            generation_config=task_cfg.generation_config,
-            chat_template=task_cfg.chat_template)
-
-
 def main():
+    from evalscope.arguments import parse_args
     args = parse_args()
     run_task(args)
 

evalscope/run_arena.py

@@ -10,7 +10,7 @@ from tqdm import tqdm
 
 from evalscope.constants import EvalConfigKeys
 from evalscope.evaluator.rating_eval import RatingEvaluate
-from evalscope.models.model_adapter import ChatGenerationModelAdapter
+from evalscope.models import ChatGenerationModelAdapter
 from evalscope.utils import get_obj_from_cfg
 from evalscope.utils.io_utils import dump_jsonl_data, jsonl_to_list, yaml_to_dict
 from evalscope.utils.logger import get_logger

evalscope/summarizer.py

@@ -6,7 +6,7 @@ from typing import List, Union
 
 from evalscope.config import TaskConfig, parse_task_config
 from evalscope.constants import EvalBackend
-from evalscope.tools.combine_reports import gen_table
+from evalscope.report import gen_table
 from evalscope.utils import csv_to_list, get_latest_folder_path
 from evalscope.utils.io_utils import OutputsStructure, json_to_dict, yaml_to_dict
 from evalscope.utils.logger import get_logger

evalscope/utils/__init__.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.constants import *
+from evalscope.utils.model_utils import EvalBackend
 from evalscope.utils.utils import *

evalscope/utils/chat_service.py

@@ -3,11 +3,9 @@ import time
 import torch
 from contextlib import contextmanager
 from functools import partial
-from modelscope import AutoModelForCausalLM, AutoTokenizer
 from pydantic import BaseModel, Field
 from threading import Thread
-from transformers import TextIteratorStreamer
-from typing import List, Literal, Optional, Union
+from typing import Any, List, Literal, Optional, Union
 
 
 class Usage(BaseModel):
@@ -66,7 +64,7 @@ class ChatCompletionResponseStreamChoice(BaseModel):
 class ChatCompletionResponse(BaseModel):
     model: str
     object: Literal['chat.completion', 'chat.completion.chunk']
-    choices: List[Union[ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice]]
+    choices: List[Union[ChatCompletionResponseChoice, ChatCompletionResponseStreamChoice, Any]]
     created: Optional[int] = Field(default_factory=lambda: int(time.time()))
     usage: Optional[Usage]
 
@@ -96,6 +94,9 @@ class TextCompletionResponse(BaseModel):
 class ChatService:
 
     def __init__(self, model_path, attn_implementation):
+        from modelscope import AutoModelForCausalLM, AutoTokenizer
+        from transformers import TextIteratorStreamer
+
         self.tokenizer = AutoTokenizer.from_pretrained(model_path, trust_remote_code=True)
         self.model = AutoModelForCausalLM.from_pretrained(
             model_path,

evalscope/utils/io_utils.py

@@ -160,3 +160,11 @@ def are_paths_same(path1, path2):
     real_path2 = os.path.realpath(os.path.abspath(os.path.expanduser(path2)))
 
     return real_path1 == real_path2
+
+
+def dict_to_json(d: dict, json_file: str):
+    """
+    Dump dict to json file.
+    """
+    with open(json_file, 'w') as f:
+        json.dump(d, f, indent=4, ensure_ascii=False)

evalscope/utils/logger.py

@@ -14,6 +14,11 @@ DEFAULT_LEVEL = logging.DEBUG if os.getenv('LOG_LEVEL', 'INFO') == 'DEBUG' else
 
 logging.basicConfig(format=simple_format, level=DEFAULT_LEVEL)
 
+# disable datasets logging
+logging.getLogger('datasets').setLevel(logging.WARNING)
+logging.getLogger('modelscope').setLevel(logging.WARNING)
+logging.getLogger('httpx').setLevel(logging.WARNING)
+
 
 def get_logger(log_file: Optional[str] = None, log_level: int = DEFAULT_LEVEL, file_mode: str = 'w', force=False):
     """Get logging logger

evalscope/utils/model_utils.py

@@ -1,7 +1,20 @@
-from transformers import GenerationConfig
+from enum import Enum
+from typing import TYPE_CHECKING
 
+if TYPE_CHECKING:
+    from transformers import GenerationConfig
 
-def fix_do_sample_warning(generation_config: GenerationConfig) -> None:
+
+class EvalBackend(Enum):
+    #  NOTE: compatible with ms-swfit v2.x
+    NATIVE = 'Native'
+    OPEN_COMPASS = 'OpenCompass'
+    VLM_EVAL_KIT = 'VLMEvalKit'
+    RAG_EVAL = 'RAGEval'
+    THIRD_PARTY = 'ThirdParty'
+
+
+def fix_do_sample_warning(generation_config: 'GenerationConfig') -> None:
     # Use the default values of temperature/top_p/top_k in generation_config.
     if generation_config.temperature == 0:
         generation_config.do_sample = False

evalscope/utils/utils.py

@@ -121,7 +121,6 @@ class ResponseParser:
             f'([{options_concat}])\s?是正确答案',
             f'选项\s?([{options_concat}])\s?正确',
             f'所以答\s?([{options_concat}])',
-            f'1.\s?([{options_concat}])[.。$]?$',
             f'所以\s?([{options_concat}][.。$]?$)',
             f'所有\s?([{options_concat}][.。$]?$)',
             f'[\s，：:,]([{options_concat}])[。，,\.]?$',
@@ -137,16 +136,15 @@ class ResponseParser:
             f'答案为(.*?)[{options_concat}]',
             f'固选(.*?)[{options_concat}]',
             f'答案应该是(.*?)[{options_concat}]',
-            f'[Tt]he answer is [{options_concat}]',
+            f'[Tt]he answer is \(?[{options_concat}]\)?',
             f'[Tt]he correct answer is [{options_concat}]',
             f'[Tt]he correct answer is:\n[{options_concat}]',
             f'(\s|^)[{options_concat}][\s。，,\.$]',  # noqa
-            f'[{options_concat}]',
             f'^选项\s?([{options_concat}])',
             f'^([{options_concat}])\s?选?项',
             f'(\s|^)[{options_concat}][\s。，,：:\.$]',
             f'(\s|^)[{options_concat}](\s|$)',
-            f'1.\s?(.*?)$',
+            f'[{options_concat}]',
         ]
 
         regexes = [re.compile(pattern) for pattern in patterns]
@@ -169,6 +167,7 @@ class ResponseParser:
         """
         patterns = [
             r'[Aa]nswer:\s*(\w+)',
+            r'answer is \(?(\w+)\)?',
             r'[Tt]he correct answer is:\s*(\w+)',
             r'[Tt]he correct answer is:\n\s*(\w+)',
             r'[Tt]he correct answer is:\n\n-\s*(\w+)',
@@ -199,27 +198,6 @@ class ResponseParser:
 
 
 
-def import_module_util(import_path_prefix: str, module_name: str, members_to_import: list) -> dict:
-    """
-    Import module utility function.
-
-    Args:
-        import_path_prefix: e.g. 'evalscope.benchmarks.'
-        module_name: The module name to import. e.g. 'mmlu'
-        members_to_import: The members to import.
-            e.g. ['DATASET_ID', 'SUBJECT_MAPPING', 'SUBSET_LIST', 'DataAdapterClass']
-
-    Returns:
-        dict: imported modules map. e.g. {'DATASET_ID': 'mmlu', 'SUBJECT_MAPPING': {...}, ...}
-    """
-    imported_modules = {}
-    module = importlib.import_module(import_path_prefix + module_name)
-    for member_name in members_to_import:
-        imported_modules[member_name] = getattr(module, member_name)
-
-    return imported_modules
-
-
 def normalize_score(score: Union[float, dict], keep_num: int = 4) -> Union[float, dict]:
     """
     Normalize score.

evalscope/version.py

@@ -1,4 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-__version__ = '0.8.2'
-__release_datetime__ = '2024-12-26 20:00:00'
+__version__ = '0.10.0'
+__release_datetime__ = '2025-01-20 20:00:00'