evalscope 0.7.2__py3-none-any.whl → 0.8.1__py3-none-any.whl

evalscope/cli/start_server.py

@@ -1,67 +1,56 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-import os, sys, time
-from argparse import ArgumentParser
+import os
 import subprocess
-
+import sys
+import time
+from argparse import ArgumentParser
 
 from evalscope.cli.base import CLICommand
 
-
 current_path = os.path.dirname(os.path.abspath(__file__))
 print(current_path)
 root_path = os.path.dirname(current_path)
 print(root_path)
 
+
 def subparser_func(args):
     """ Function which will be called for a specific sub parser.
     """
     return PerfServerCMD(args)
 
+
 def add_perf_args(parser):
+    parser.add_argument('--server-command', required=True, type=str, help='The start server command.')
     parser.add_argument(
-        '--server-command', required=True, type=str, help='The start server command.')
-    parser.add_argument(
-        '--logdir', required=True, type=str, help='The monitor log save dir, tensorboard start at this path for display!')
-    parser.add_argument(
-        '--host', type=str, default='0.0.0.0', help='The tensorboard host'
-    )    
-    parser.add_argument(
-        '--tensorboard-port', type=str, default='6006', help='The tensorboard port'
-    )
+        '--logdir',
+        required=True,
+        type=str,
+        help='The monitor log save dir, tensorboard start at this path for display!')
+    parser.add_argument('--host', type=str, default='0.0.0.0', help='The tensorboard host')
+    parser.add_argument('--tensorboard-port', type=str, default='6006', help='The tensorboard port')
+
 
 def async_run_command_with_popen(cmd):
     sub_process = subprocess.Popen(
-        cmd,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
-        bufsize=1,
-        universal_newlines=True,
-        encoding='utf8')
+        cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=1, universal_newlines=True, encoding='utf8')
     return sub_process
 
+
 def start_monitor(args):
-    cmd = ['python',
-           '%s/perf/monitor.py'%root_path,
-           '--logdir',
-           args.logdir]
+    cmd = ['python', '%s/perf/monitor.py' % root_path, '--logdir', args.logdir]
     print(cmd)
     p = async_run_command_with_popen(cmd)
     os.set_blocking(p.stdout.fileno(), False)
     return p
 
+
 def start_tensorboard(args):
-    cmd = ['tensorboard',
-           '--logdir',
-           args.logdir,
-           '--host',
-           args.host,
-           '--port',
-           args.tensorboard_port
-           ]
+    cmd = ['tensorboard', '--logdir', args.logdir, '--host', args.host, '--port', args.tensorboard_port]
     p = async_run_command_with_popen(cmd)
     os.set_blocking(p.stdout.fileno(), False)
     return p
 
+
 def start_server(args):
     cmd = args.server_command
     print(cmd)
@@ -76,7 +65,7 @@ def start_server(args):
 
     os.set_blocking(sub_process.stdout.fileno(), False)
     return sub_process
- 
+
 
 def wait_for_workers(workers):
     while True:
@@ -91,12 +80,12 @@ def wait_for_workers(workers):
                     else:
                         break
             else:
-                print('Worker %s completed!'%idx)
+                print('Worker %s completed!' % idx)
                 for line in iter(worker.stdout.readline, ''):
                     if line != '':
                         sys.stdout.write(line)
                     else:
-                        break                
+                        break
                 workers[idx] = None
 
         is_all_completed = True
@@ -108,7 +97,8 @@ def wait_for_workers(workers):
         if is_all_completed:
             break
         time.sleep(0.1)
-        
+
+
 class PerfServerCMD(CLICommand):
     name = 'server'
 
@@ -127,12 +117,8 @@ class PerfServerCMD(CLICommand):
         # start monitor
         p_monitor = start_monitor(self.args)
         # start tensorboard
-        p_tensorboard = start_tensorboard(self.args)        
+        p_tensorboard = start_tensorboard(self.args)
         # start server
         p_server = start_server(self.args)
-        
+
         wait_for_workers([p_monitor, p_tensorboard, p_server])
-        
-        
-        
-        

evalscope/config.py

@@ -1,69 +1,137 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import os
 import copy
-from dataclasses import dataclass, asdict, field
-from typing import Optional, List
+import json
+import os
+from argparse import Namespace
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Union
 
-from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
+from evalscope.constants import DEFAULT_DATASET_CACHE_DIR, DEFAULT_WORK_DIR, EvalBackend, EvalStage, EvalType, HubType
 from evalscope.models.custom import CustomModel
-from evalscope.utils import yaml_to_dict
+from evalscope.utils import gen_hash
+from evalscope.utils.io_utils import dict_to_yaml, json_to_dict, yaml_to_dict
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 cur_path = os.path.dirname(os.path.abspath(__file__))
 
-registry_tasks = {
-    'arc': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/arc.yaml')),
-    'gsm8k': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/gsm8k.yaml')),
-    'mmlu': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu.yaml')),
-    'cmmlu': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/cmmlu.yaml')),
-    'ceval': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval.yaml')),
-    'bbh': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh.yaml')),
-    'general_qa': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/general_qa.yaml')),
-
-    # 'bbh_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh_mini.yaml')),
-    # 'mmlu_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu_mini.yaml')),
-    # 'ceval_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval_mini.yaml')),
-
+DEFAULT_MODEL_ARGS = {'revision': 'master', 'precision': 'torch.float16', 'device': 'auto'}
+DEFAULT_GENERATION_CONFIG = {
+    'max_length': 2048,
+    'max_new_tokens': 512,
+    'do_sample': False,
+    'top_k': 50,
+    'top_p': 1.0,
+    'temperature': 1.0,
 }
 
 
 @dataclass
 class TaskConfig:
-    model_args: Optional[dict] = field(default_factory=dict)
-    template_type: Optional[str] = 'default-generation'
-    generation_config: Optional[dict] = field(default_factory=dict)
-    dataset_args: Optional[dict] = field(default_factory=dict)
+    # Model-related arguments
+    model: Union[str, CustomModel, None] = None
+    model_id: Optional[str] = None
+    model_args: Optional[Dict] = field(default_factory=lambda: DEFAULT_MODEL_ARGS | {})
+
+    # Template-related arguments
+    template_type: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
+    chat_template: Optional[str] = None
+
+    # Dataset-related arguments
+    datasets: Optional[List[str]] = None
+    dataset_args: Optional[Dict] = field(default_factory=dict)
+    dataset_dir: str = DEFAULT_DATASET_CACHE_DIR
+    dataset_hub: str = HubType.MODELSCOPE
+
+    # Generation configuration arguments
+    generation_config: Optional[Dict] = field(default_factory=lambda: DEFAULT_GENERATION_CONFIG | {})
+
+    # Evaluation-related arguments
+    eval_type: str = EvalType.CHECKPOINT
+    eval_backend: str = EvalBackend.NATIVE
+    eval_config: Union[str, Dict, None] = None
+    stage: str = EvalStage.ALL
+    limit: Optional[int] = None
+
+    # Cache and working directory arguments
+    mem_cache: bool = False  # Deprecated, will be removed in v1.0.0.
+    use_cache: Optional[str] = None
+    work_dir: str = DEFAULT_WORK_DIR
+    outputs: Optional[str] = None  # Deprecated, will be removed in v1.0.0.
+
+    # Debug and runtime mode arguments
+    debug: bool = False
     dry_run: bool = False
-    model: CustomModel = None
-    eval_type: str = 'custom'
-    datasets: list = field(default_factory=list)
-    work_dir: str = DEFAULT_ROOT_CACHE_DIR
-    outputs: str = None
-    mem_cache: bool = False
-    use_cache: bool = True
-    stage: str = 'all'      # `all` or `infer` or `review`
-    dataset_hub: str = 'ModelScope'
-    dataset_dir: str = DEFAULT_ROOT_CACHE_DIR
-    limit: int = None
-    eval_backend: str = 'Native'
-    eval_config: dict = field(default_factory=dict)
-
-    # def __post_init__(self):
-    #     self.registry_tasks = {
-    #         'arc': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/arc.yaml')),
-    #         'gsm8k': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/gsm8k.yaml')),
-    #         'mmlu': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu.yaml')),
-    #         'ceval': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval.yaml')),
-    #         'bbh': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh.yaml')),
-    #
-    #         'bbh_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh_mini.yaml')),
-    #         'mmlu_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu_mini.yaml')),
-    #         'ceval_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval_mini.yaml')),
-    #
-    #     }
+    seed: int = 42
+
+    def __post_init__(self):
+        if (not self.model_id) and self.model:
+            if isinstance(self.model, CustomModel):
+                self.model_id = type(self.model).__name__
+            else:
+                self.model_id = os.path.basename(self.model).rstrip(os.sep)
+
+    def to_dict(self):
+        # Note: to avoid serialization error for some model instance
+        return self.__dict__
+
+    def __str__(self):
+        return json.dumps(self.to_dict(), indent=4, default=str, ensure_ascii=False)
+
+    def update(self, other: Union['TaskConfig', dict]):
+        if isinstance(other, TaskConfig):
+            other = other.to_dict()
+        self.__dict__.update(other)
+
+    def dump_yaml(self, output_dir: str):
+        """Dump the task configuration to a YAML file."""
+        task_cfg_file = os.path.join(output_dir, f'task_config_{gen_hash(str(self), bits=6)}.yaml')
+        try:
+            logger.info(f'Dump task config to {task_cfg_file}')
+            dict_to_yaml(self.to_dict(), task_cfg_file)
+        except Exception as e:
+            logger.warning(f'Failed to dump overall task config: {e}')
+
+    @staticmethod
+    def list():
+        return list(registry_tasks.keys())
+
+    @staticmethod
+    def from_yaml(yaml_file: str):
+        return TaskConfig.from_dict(yaml_to_dict(yaml_file))
+
+    @staticmethod
+    def from_dict(d: dict):
+        return TaskConfig(**d)
+
+    @staticmethod
+    def from_json(json_file: str):
+        return TaskConfig.from_dict(json_to_dict(json_file))
+
+    @staticmethod
+    def from_args(args: Namespace):
+        # Convert Namespace to a dictionary and filter out None values
+        args_dict = {k: v for k, v in vars(args).items() if v is not None}
+        del args_dict['func']  # Note: compat CLI arguments
+
+        return TaskConfig.from_dict(args_dict)
+
+    @staticmethod
+    def load(custom_model: CustomModel, tasks: List[str]) -> List['TaskConfig']:
+        res_list = []
+        for task_name in tasks:
+            task = registry_tasks.get(task_name, None)
+            if task is None:
+                logger.error(f'No task found in tasks: {list(registry_tasks.keys())}, got task_name: {task_name}')
+                continue
+
+            task.model = custom_model
+            task.model_id = type(custom_model).__name__
+            res_list.append(task)
+
+        return res_list
 
     @staticmethod
     def registry(name: str, data_pattern: str, dataset_dir: str = None, subset_list: list = None) -> None:
@@ -75,7 +143,7 @@ class TaskConfig:
             data_pattern: str, the data pattern for the task.
                     e.g. `mmlu`, `ceval`, `gsm8k`, ...
                     refer to task_config.list() for all available datasets.
-            dataset_dir: str, the directory to store multiple datasets files. e.g. /path/to/data, 
+            dataset_dir: str, the directory to store multiple datasets files. e.g. /path/to/data,
                 then your specific custom dataset directory will be /path/to/data/{name}
             subset_list: list, the subset list for the dataset.
                 e.g. ['middle_school_politics', 'operating_system']
@@ -83,63 +151,55 @@ class TaskConfig:
         """
         available_datasets = list(registry_tasks.keys())
         if data_pattern not in available_datasets:
-            logger.error(f'No dataset found in available datasets: {available_datasets}, got data_pattern: {data_pattern}')
+            logger.error(
+                f'No dataset found in available datasets: {available_datasets}, got data_pattern: {data_pattern}')
             return
 
         # Reuse the existing task config and update the datasets
-        pattern_config = registry_tasks.get(data_pattern)
+        pattern_config = registry_tasks[data_pattern]
 
         custom_config = copy.deepcopy(pattern_config)
-        custom_config.update({'datasets': [data_pattern]})
-        custom_config.update({'dataset_hub': 'Local'})     # TODO: to support `ModelScope`
-        if 'dataset_args' in custom_config:
-            if data_pattern not in custom_config:
-                custom_config['dataset_args'].update({data_pattern: {}})
-        else:
-            custom_config.update({'dataset_args': {data_pattern: {}}})
+        custom_config.datasets = [data_pattern]
+        custom_config.dataset_args = {data_pattern: {}}
+        custom_config.eval_type = EvalType.CHECKPOINT
 
         if dataset_dir is not None:
-            custom_config['dataset_args'][data_pattern].update({'local_path': dataset_dir})
+            custom_config.dataset_args[data_pattern].update({'local_path': dataset_dir})
 
         if subset_list is not None:
-            # custom_config['dataset_args'].get(data_pattern, {}).update({'subset_list': subset_list})
-            custom_config['dataset_args'][data_pattern].update({'subset_list': subset_list})
+            custom_config.dataset_args[data_pattern].update({'subset_list': subset_list})
 
         registry_tasks.update({name: custom_config})
         logger.info(f'** Registered task: {name} with data pattern: {data_pattern}')
 
-    def to_dict(self):
-        # Note: to avoid serialization error for some model instance
-        _tmp_model = copy.copy(self.model)
-        self.model = None
-        res_dict = asdict(self)
-        res_dict.update({'model': _tmp_model})
-        self.model = _tmp_model
-
-        return res_dict
 
-    @staticmethod
-    def load(custom_model: CustomModel, tasks: List[str]) -> List['TaskConfig']:
-        res_list = []
-        for task_name in tasks:
-            task: dict = registry_tasks.get(task_name, None)
-            if task is None:
-                logger.error(f'No task found in tasks: {list(registry_tasks.keys())}, got task_name: {task_name}')
-                continue
-
-            res = TaskConfig(**task)
-            res.model = custom_model
-            if res.outputs is None:
-                res.outputs = os.path.join(res.work_dir,
-                                           'outputs',
-                                           f"eval_{'-'.join(tasks)}_{res.model.config['model_id']}_{res.model_args.get('revision', 'default')}")
-            res_list.append(res)
-
-        return res_list
-
-    @staticmethod
-    def list():
-        return list(registry_tasks.keys())
+tasks = ['arc', 'gsm8k', 'mmlu', 'cmmlu', 'ceval', 'bbh', 'general_qa']
+
+registry_tasks = {task: TaskConfig.from_yaml(os.path.join(cur_path, f'registry/tasks/{task}.yaml')) for task in tasks}
+
+
+def parse_task_config(task_cfg) -> TaskConfig:
+    """Parse task configuration from various formats into a TaskConfig object."""
+    if isinstance(task_cfg, TaskConfig):
+        logger.info('Args: Task config is provided with TaskConfig type.')
+    elif isinstance(task_cfg, dict):
+        logger.info('Args: Task config is provided with dictionary type.')
+        task_cfg = TaskConfig.from_dict(task_cfg)
+    elif isinstance(task_cfg, Namespace):
+        logger.info('Args: Task config is provided with CommandLine type.')
+        task_cfg = TaskConfig.from_args(task_cfg)
+    elif isinstance(task_cfg, str):
+        extension = task_cfg.split('.')[-1]
+        logger.info(f'Args: Task config is provided with {extension} file type.')
+        if extension in ['yaml', 'yml']:
+            task_cfg = TaskConfig.from_yaml(task_cfg)
+        elif extension == 'json':
+            task_cfg = TaskConfig.from_json(task_cfg)
+        else:
+            raise ValueError('Args: Unsupported file extension.')
+    else:
+        raise ValueError('Args: Please provide a valid task config.')
+    return task_cfg
 
 
 class TempModel(CustomModel):
@@ -158,9 +218,7 @@ if __name__ == '__main__':
     # Register a new task
     TaskConfig.registry(name='arc_swift', data_pattern='arc', dataset_dir='/path/to/swift_custom_work')
 
-    import json
     swift_eval_task: List[TaskConfig] = TaskConfig.load(custom_model=model, tasks=['gsm8k', 'arc', 'arc_swift'])
     for item in swift_eval_task:
-        print(item.to_dict())
+        print(item)
         print()
-

evalscope/constants.py

@@ -1,7 +1,18 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
-from enum import Enum
+from modelscope.utils.constant import DEFAULT_REPOSITORY_REVISION
+from modelscope.utils.file_utils import get_dataset_cache_root, get_model_cache_root
 
-DEFAULT_ROOT_CACHE_DIR = '~/.cache/evalscope'
+DEFAULT_WORK_DIR = './outputs'
+DEFAULT_MODEL_REVISION = DEFAULT_REPOSITORY_REVISION  # master
+DEFAULT_MODEL_CACHE_DIR = get_model_cache_root()  # ~/.cache/modelscope/hub
+DEFAULT_DATASET_CACHE_DIR = get_dataset_cache_root()  # ~/.cache/modelscope/datasets
+DEFAULT_ROOT_CACHE_DIR = DEFAULT_DATASET_CACHE_DIR  # compatible with old version
+
+
+class HubType:
+    MODELSCOPE = 'modelscope'
+    HUGGINGFACE = 'huggingface'
+    LOCAL = 'local'
 
 
 class DumpMode:
@@ -25,7 +36,7 @@ class MetricsConstant:
     ]
 
 
-class MetricMembers(Enum):
+class MetricMembers:
 
     # Math accuracy metric
     MATH_ACCURACY = 'math_accuracy'
@@ -65,54 +76,25 @@ class ArenaMode:
     PAIRWISE_BASELINE = 'pairwise_baseline'
 
 
-class OutputsStructure:
-
-    LOGS_DIR = 'logs_dir'
-
-    PREDICTIONS_DIR = 'predictions_dir'
-
-    REVIEWS_DIR = 'reviews_dir'
-
-    REPORTS_DIR = 'reports_dir'
-
-    CONFIGS_DIR = 'configs_dir'
-
-
 class AnswerKeys:
-
     ANSWER_ID = 'answer_id'
-
     RAW_INPUT = 'raw_input'
-
     ORIGIN_PROMPT = 'origin_prompt'
-
     MODEL_SPEC = 'model_spec'
-
     SUBSET_NAME = 'subset_name'
-
     CHOICES = 'choices'
 
 
 class ReviewKeys:
-
     REVIEW_ID = 'review_id'
-
     REVIEWED = 'reviewed'
-
     REVIEWER_SPEC = 'reviewer_spec'
-
     REVIEW_TIME = 'review_time'
-
     MESSAGE = 'message'
-
     CONTENT = 'content'
-
     GOLD = 'gold'
-
     PRED = 'pred'
-
     RESULT = 'result'
-
     REVIEW = 'review'
 
 
@@ -148,3 +130,39 @@ class EvalStage:
     ALL = 'all'
     INFER = 'infer'
     REVIEW = 'review'
+
+
+class EvalType:
+
+    CUSTOM = 'custom'
+    CHECKPOINT = 'checkpoint'
+
+
+class EvalBackend:
+
+    class _Backend:
+        #  compatible with old version, set 'value'
+
+        def __init__(self, value):
+            self._value = value
+
+        @property
+        def value(self):
+            return self._value
+
+        def __str__(self):
+            return self._value
+
+        def __repr__(self):
+            return f"'{self._value}'"
+
+        def __eq__(self, other):
+            if isinstance(other, str):
+                return self._value == other
+            return NotImplemented
+
+    NATIVE = _Backend('Native')
+    OPEN_COMPASS = _Backend('OpenCompass')
+    VLM_EVAL_KIT = _Backend('VLMEvalKit')
+    RAG_EVAL = _Backend('RAGEval')
+    THIRD_PARTY = _Backend('ThirdParty')