evalscope 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/config.py

@@ -1,69 +1,127 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import os
 import copy
-from dataclasses import dataclass, asdict, field
-from typing import Optional, List
+import json
+import os
+from argparse import Namespace
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Union
 
-from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
+from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, DEFAULT_WORK_DIR, EvalBackend, EvalStage, EvalType, HubType
 from evalscope.models.custom import CustomModel
-from evalscope.utils import yaml_to_dict
+from evalscope.utils import dict_to_yaml, gen_hash, json_to_dict, yaml_to_dict
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 cur_path = os.path.dirname(os.path.abspath(__file__))
 
-registry_tasks = {
-    'arc': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/arc.yaml')),
-    'gsm8k': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/gsm8k.yaml')),
-    'mmlu': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu.yaml')),
-    'cmmlu': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/cmmlu.yaml')),
-    'ceval': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval.yaml')),
-    'bbh': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh.yaml')),
-    'general_qa': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/general_qa.yaml')),
-
-    # 'bbh_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh_mini.yaml')),
-    # 'mmlu_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu_mini.yaml')),
-    # 'ceval_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval_mini.yaml')),
-
+DEFAULT_MODEL_ARGS = {'revision': 'master', 'precision': 'torch.float16', 'device': 'auto'}
+DEFAULT_GENERATION_CONFIG = {
+    'max_length': 2048,
+    'max_new_tokens': 512,
+    'do_sample': False,
+    'top_k': 50,
+    'top_p': 1.0,
+    'temperature': 1.0,
 }
 
 
 @dataclass
 class TaskConfig:
-    model_args: Optional[dict] = field(default_factory=dict)
-    template_type: Optional[str] = 'default-generation'
-    generation_config: Optional[dict] = field(default_factory=dict)
-    dataset_args: Optional[dict] = field(default_factory=dict)
+    # Model-related arguments
+    model: Union[str, CustomModel, None] = None
+    model_args: Optional[Dict] = field(default_factory=lambda: DEFAULT_MODEL_ARGS | {})
+
+    # Template-related arguments
+    template_type: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
+    chat_template: Optional[str] = None
+
+    # Dataset-related arguments
+    datasets: Optional[List[str]] = None
+    dataset_args: Optional[Dict] = field(default_factory=dict)
+    dataset_dir: str = DEFAULT_DATASET_CACHE_DIR
+    dataset_hub: str = HubType.MODELSCOPE
+
+    # Generation configuration arguments
+    generation_config: Optional[Dict] = field(default_factory=lambda: DEFAULT_GENERATION_CONFIG | {})
+
+    # Evaluation-related arguments
+    eval_type: str = EvalType.CHECKPOINT
+    eval_backend: str = EvalBackend.NATIVE
+    eval_config: Union[str, Dict, None] = None
+    stage: str = EvalStage.ALL
+    limit: Optional[int] = None
+
+    # Cache and working directory arguments
+    mem_cache: bool = False  # Deprecated, will be removed in v1.0.0.
+    use_cache: Optional[str] = None
+    work_dir: str = DEFAULT_WORK_DIR
+    outputs: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
+
+    # Debug and runtime mode arguments
+    debug: bool = False
     dry_run: bool = False
-    model: CustomModel = None
-    eval_type: str = 'custom'
-    datasets: list = field(default_factory=list)
-    work_dir: str = DEFAULT_ROOT_CACHE_DIR
-    outputs: str = None
-    mem_cache: bool = False
-    use_cache: bool = True
-    stage: str = 'all'      # `all` or `infer` or `review`
-    dataset_hub: str = 'ModelScope'
-    dataset_dir: str = DEFAULT_ROOT_CACHE_DIR
-    limit: int = None
-    eval_backend: str = 'Native'
-    eval_config: dict = field(default_factory=dict)
-
-    # def __post_init__(self):
-    #     self.registry_tasks = {
-    #         'arc': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/arc.yaml')),
-    #         'gsm8k': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/gsm8k.yaml')),
-    #         'mmlu': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu.yaml')),
-    #         'ceval': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval.yaml')),
-    #         'bbh': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh.yaml')),
-    #
-    #         'bbh_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh_mini.yaml')),
-    #         'mmlu_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu_mini.yaml')),
-    #         'ceval_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval_mini.yaml')),
-    #
-    #     }
+    seed: int = 42
+
+    def to_dict(self):
+        # Note: to avoid serialization error for some model instance
+        return self.__dict__
+
+    def __str__(self):
+        return json.dumps(self.to_dict(), indent=4, default=str, ensure_ascii=False)
+
+    def update(self, other: Union['TaskConfig', dict]):
+        if isinstance(other, TaskConfig):
+            other = other.to_dict()
+        self.__dict__.update(other)
+
+    def dump_yaml(self, output_dir: str):
+        """Dump the task configuration to a YAML file."""
+        task_cfg_file = os.path.join(output_dir, f'task_config_{gen_hash(str(self), bits=6)}.yaml')
+        try:
+            logger.info(f'Dump task config to {task_cfg_file}')
+            dict_to_yaml(self.to_dict(), task_cfg_file)
+        except Exception as e:
+            logger.warning(f'Failed to dump overall task config: {e}')
+
+    @staticmethod
+    def list():
+        return list(registry_tasks.keys())
+
+    @staticmethod
+    def from_yaml(yaml_file: str):
+        return TaskConfig.from_dict(yaml_to_dict(yaml_file))
+
+    @staticmethod
+    def from_dict(d: dict):
+        return TaskConfig(**d)
+
+    @staticmethod
+    def from_json(json_file: str):
+        return TaskConfig.from_dict(json_to_dict(json_file))
+
+    @staticmethod
+    def from_args(args: Namespace):
+        # Convert Namespace to a dictionary and filter out None values
+        args_dict = {k: v for k, v in vars(args).items() if v is not None}
+        del args_dict['func']  # Note: compat CLI arguments
+
+        return TaskConfig.from_dict(args_dict)
+
+    @staticmethod
+    def load(custom_model: CustomModel, tasks: List[str]) -> List['TaskConfig']:
+        res_list = []
+        for task_name in tasks:
+            task = registry_tasks.get(task_name, None)
+            if task is None:
+                logger.error(f'No task found in tasks: {list(registry_tasks.keys())}, got task_name: {task_name}')
+                continue
+
+            task.model = custom_model
+            res_list.append(task)
+
+        return res_list
 
     @staticmethod
     def registry(name: str, data_pattern: str, dataset_dir: str = None, subset_list: list = None) -> None:
@@ -75,7 +133,7 @@ class TaskConfig:
             data_pattern: str, the data pattern for the task.
                     e.g. `mmlu`, `ceval`, `gsm8k`, ...
                     refer to task_config.list() for all available datasets.
-            dataset_dir: str, the directory to store multiple datasets files. e.g. /path/to/data, 
+            dataset_dir: str, the directory to store multiple datasets files. e.g. /path/to/data,
                 then your specific custom dataset directory will be /path/to/data/{name}
             subset_list: list, the subset list for the dataset.
                 e.g. ['middle_school_politics', 'operating_system']
@@ -83,63 +141,31 @@ class TaskConfig:
         """
         available_datasets = list(registry_tasks.keys())
         if data_pattern not in available_datasets:
-            logger.error(f'No dataset found in available datasets: {available_datasets}, got data_pattern: {data_pattern}')
+            logger.error(
+                f'No dataset found in available datasets: {available_datasets}, got data_pattern: {data_pattern}')
             return
 
         # Reuse the existing task config and update the datasets
-        pattern_config = registry_tasks.get(data_pattern)
+        pattern_config = registry_tasks[data_pattern]
 
         custom_config = copy.deepcopy(pattern_config)
-        custom_config.update({'datasets': [data_pattern]})
-        custom_config.update({'dataset_hub': 'Local'})     # TODO: to support `ModelScope`
-        if 'dataset_args' in custom_config:
-            if data_pattern not in custom_config:
-                custom_config['dataset_args'].update({data_pattern: {}})
-        else:
-            custom_config.update({'dataset_args': {data_pattern: {}}})
+        custom_config.datasets = [data_pattern]
+        custom_config.dataset_args = {data_pattern: {}}
+        custom_config.eval_type = EvalType.CHECKPOINT
 
         if dataset_dir is not None:
-            custom_config['dataset_args'][data_pattern].update({'local_path': dataset_dir})
+            custom_config.dataset_args[data_pattern].update({'local_path': dataset_dir})
 
         if subset_list is not None:
-            # custom_config['dataset_args'].get(data_pattern, {}).update({'subset_list': subset_list})
-            custom_config['dataset_args'][data_pattern].update({'subset_list': subset_list})
+            custom_config.dataset_args[data_pattern].update({'subset_list': subset_list})
 
         registry_tasks.update({name: custom_config})
         logger.info(f'** Registered task: {name} with data pattern: {data_pattern}')
 
-    def to_dict(self):
-        # Note: to avoid serialization error for some model instance
-        _tmp_model = copy.copy(self.model)
-        self.model = None
-        res_dict = asdict(self)
-        res_dict.update({'model': _tmp_model})
-        self.model = _tmp_model
 
-        return res_dict
+tasks = ['arc', 'gsm8k', 'mmlu', 'cmmlu', 'ceval', 'bbh', 'general_qa']
 
-    @staticmethod
-    def load(custom_model: CustomModel, tasks: List[str]) -> List['TaskConfig']:
-        res_list = []
-        for task_name in tasks:
-            task: dict = registry_tasks.get(task_name, None)
-            if task is None:
-                logger.error(f'No task found in tasks: {list(registry_tasks.keys())}, got task_name: {task_name}')
-                continue
-
-            res = TaskConfig(**task)
-            res.model = custom_model
-            if res.outputs is None:
-                res.outputs = os.path.join(res.work_dir,
-                                           'outputs',
-                                           f"eval_{'-'.join(tasks)}_{res.model.config['model_id']}_{res.model_args.get('revision', 'default')}")
-            res_list.append(res)
-
-        return res_list
-
-    @staticmethod
-    def list():
-        return list(registry_tasks.keys())
+registry_tasks = {task: TaskConfig.from_yaml(os.path.join(cur_path, f'registry/tasks/{task}.yaml')) for task in tasks}
 
 
 class TempModel(CustomModel):
@@ -158,9 +184,7 @@ if __name__ == '__main__':
     # Register a new task
     TaskConfig.registry(name='arc_swift', data_pattern='arc', dataset_dir='/path/to/swift_custom_work')
 
-    import json
     swift_eval_task: List[TaskConfig] = TaskConfig.load(custom_model=model, tasks=['gsm8k', 'arc', 'arc_swift'])
     for item in swift_eval_task:
-        print(item.to_dict())
+        print(item)
         print()
-

evalscope/constants.py

@@ -1,7 +1,18 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-from enum import Enum
+import os
+from modelscope.utils.constant import DEFAULT_REPOSITORY_REVISION
+from modelscope.utils.file_utils import get_dataset_cache_root, get_model_cache_root
 
-DEFAULT_ROOT_CACHE_DIR = '~/.cache/evalscope'
+DEFAULT_WORK_DIR = './outputs'
+DEFAULT_MODEL_REVISION = DEFAULT_REPOSITORY_REVISION  # master
+DEFAULT_MODEL_CACHE_DIR = get_model_cache_root()  # ~/.cache/modelscope/hub
+DEFAULT_DATASET_CACHE_DIR = get_dataset_cache_root()  # ~/.cache/modelscope/datasets
+
+
+class HubType:
+    MODELSCOPE = 'modelscope'
+    HUGGINGFACE = 'huggingface'
+    LOCAL = 'local'
 
 
 class DumpMode:
@@ -25,7 +36,7 @@ class MetricsConstant:
     ]
 
 
-class MetricMembers(Enum):
+class MetricMembers:
 
     # Math accuracy metric
     MATH_ACCURACY = 'math_accuracy'
@@ -66,53 +77,51 @@ class ArenaMode:
 
 
 class OutputsStructure:
-
-    LOGS_DIR = 'logs_dir'
-
-    PREDICTIONS_DIR = 'predictions_dir'
-
-    REVIEWS_DIR = 'reviews_dir'
-
-    REPORTS_DIR = 'reports_dir'
-
-    CONFIGS_DIR = 'configs_dir'
+    LOGS_DIR = 'logs'
+    PREDICTIONS_DIR = 'predictions'
+    REVIEWS_DIR = 'reviews'
+    REPORTS_DIR = 'reports'
+    CONFIGS_DIR = 'configs'
+
+    def __init__(self, outputs_dir: str, is_make: bool = True):
+        self.outputs_dir = outputs_dir
+        self.logs_dir = os.path.join(outputs_dir, OutputsStructure.LOGS_DIR)
+        self.predictions_dir = os.path.join(outputs_dir, OutputsStructure.PREDICTIONS_DIR)
+        self.reviews_dir = os.path.join(outputs_dir, OutputsStructure.REVIEWS_DIR)
+        self.reports_dir = os.path.join(outputs_dir, OutputsStructure.REPORTS_DIR)
+        self.configs_dir = os.path.join(outputs_dir, OutputsStructure.CONFIGS_DIR)
+
+        if is_make:
+            self.create_directories()
+
+    def create_directories(self):
+        os.makedirs(self.outputs_dir, exist_ok=True)
+        os.makedirs(self.logs_dir, exist_ok=True)
+        os.makedirs(self.predictions_dir, exist_ok=True)
+        os.makedirs(self.reviews_dir, exist_ok=True)
+        os.makedirs(self.reports_dir, exist_ok=True)
+        os.makedirs(self.configs_dir, exist_ok=True)
 
 
 class AnswerKeys:
-
     ANSWER_ID = 'answer_id'
-
     RAW_INPUT = 'raw_input'
-
     ORIGIN_PROMPT = 'origin_prompt'
-
     MODEL_SPEC = 'model_spec'
-
     SUBSET_NAME = 'subset_name'
-
     CHOICES = 'choices'
 
 
 class ReviewKeys:
-
     REVIEW_ID = 'review_id'
-
     REVIEWED = 'reviewed'
-
     REVIEWER_SPEC = 'reviewer_spec'
-
     REVIEW_TIME = 'review_time'
-
     MESSAGE = 'message'
-
     CONTENT = 'content'
-
     GOLD = 'gold'
-
     PRED = 'pred'
-
     RESULT = 'result'
-
     REVIEW = 'review'
 
 
@@ -148,3 +157,26 @@ class EvalStage:
     ALL = 'all'
     INFER = 'infer'
     REVIEW = 'review'
+
+
+class EvalType:
+
+    CUSTOM = 'custom'
+    CHECKPOINT = 'checkpoint'
+
+
+class EvalBackend:
+    # Use native evaluation pipeline of EvalScope
+    NATIVE = 'Native'
+
+    # Use OpenCompass framework as the evaluation backend
+    OPEN_COMPASS = 'OpenCompass'
+
+    # Use VLM Eval Kit as the multi-modal model evaluation backend
+    VLM_EVAL_KIT = 'VLMEvalKit'
+
+    # Use RAGEval as the RAG evaluation backend
+    RAG_EVAL = 'RAGEval'
+
+    # Use third-party evaluation backend/modules
+    THIRD_PARTY = 'ThirdParty'

evalscope/evaluator/__init__.py

@@ -1,3 +1,4 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
 from evalscope.evaluator.evaluator import Evaluator
+from evalscope.evaluator.humaneval_evaluator import HumanevalEvaluator