evalscope 0.5.0__py3-none-any.whl

evalscope/cli/start_perf.py

@@ -0,0 +1,37 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from abc import abstractmethod
+import os, sys, time
+from argparse import ArgumentParser
+import subprocess
+
+
+from evalscope.cli.base import CLICommand
+from evalscope.perf.http_client import add_argument, run_perf_benchmark
+
+current_path = os.path.dirname(os.path.abspath(__file__))
+root_path = os.path.dirname(current_path)
+def subparser_func(args):
+    """ Function which will be called for a specific sub parser.
+    """
+    return PerfBenchCMD(args)
+        
+class PerfBenchCMD(CLICommand):
+    name = 'perf'
+
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def define_args(parsers: ArgumentParser):
+        """ define args for create pipeline template command.
+        """
+        parser = parsers.add_parser(PerfBenchCMD.name)
+        add_argument(parser)
+        parser.set_defaults(func=subparser_func)
+        
+    def execute(self):
+        run_perf_benchmark(self.args)
+        
+        
+        
+        

evalscope/cli/start_server.py

@@ -0,0 +1,138 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os, sys, time
+from argparse import ArgumentParser
+import subprocess
+
+
+from evalscope.cli.base import CLICommand
+
+
+current_path = os.path.dirname(os.path.abspath(__file__))
+print(current_path)
+root_path = os.path.dirname(current_path)
+print(root_path)
+
+def subparser_func(args):
+    """ Function which will be called for a specific sub parser.
+    """
+    return PerfServerCMD(args)
+
+def add_perf_args(parser):
+    parser.add_argument(
+        '--server-command', required=True, type=str, help='The start server command.')
+    parser.add_argument(
+        '--logdir', required=True, type=str, help='The monitor log save dir, tensorboard start at this path for display!')
+    parser.add_argument(
+        '--host', type=str, default='0.0.0.0', help='The tensorboard host'
+    )    
+    parser.add_argument(
+        '--tensorboard-port', type=str, default='6006', help='The tensorboard port'
+    )
+
+def async_run_command_with_popen(cmd):
+    sub_process = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        bufsize=1,
+        universal_newlines=True,
+        encoding='utf8')
+    return sub_process
+
+def start_monitor(args):
+    cmd = ['python',
+           '%s/perf/monitor.py'%root_path,
+           '--logdir',
+           args.logdir]
+    print(cmd)
+    p = async_run_command_with_popen(cmd)
+    os.set_blocking(p.stdout.fileno(), False)
+    return p
+
+def start_tensorboard(args):
+    cmd = ['tensorboard',
+           '--logdir',
+           args.logdir,
+           '--host',
+           args.host,
+           '--port',
+           args.tensorboard_port
+           ]
+    p = async_run_command_with_popen(cmd)
+    os.set_blocking(p.stdout.fileno(), False)
+    return p
+
+def start_server(args):
+    cmd = args.server_command
+    print(cmd)
+    sub_process = subprocess.Popen(
+        cmd,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
+        bufsize=1,
+        shell=True,
+        universal_newlines=True,
+        encoding='utf8')
+
+    os.set_blocking(sub_process.stdout.fileno(), False)
+    return sub_process
+ 
+
+def wait_for_workers(workers):
+    while True:
+        for idx, worker in enumerate(workers):
+            if worker is None:
+                continue
+            # check worker is completed.
+            if worker.poll() is None:
+                for line in iter(worker.stdout.readline, ''):
+                    if line != '':
+                        sys.stdout.write(line)
+                    else:
+                        break
+            else:
+                print('Worker %s completed!'%idx)
+                for line in iter(worker.stdout.readline, ''):
+                    if line != '':
+                        sys.stdout.write(line)
+                    else:
+                        break                
+                workers[idx] = None
+
+        is_all_completed = True
+        for idx, worker in enumerate(workers):
+            if worker is not None:
+                is_all_completed = False
+                break
+
+        if is_all_completed:
+            break
+        time.sleep(0.1)
+        
+class PerfServerCMD(CLICommand):
+    name = 'server'
+
+    def __init__(self, args):
+        self.args = args
+
+    @staticmethod
+    def define_args(parsers: ArgumentParser):
+        """ define args for create pipeline template command.
+        """
+        parser = parsers.add_parser(PerfServerCMD.name)
+        add_perf_args(parser)
+        parser.set_defaults(func=subparser_func)
+
+    def execute(self):
+        # start monitor
+        p_monitor = start_monitor(self.args)
+        # start tensorboard
+        p_tensorboard = start_tensorboard(self.args)        
+        # start server
+        p_server = start_server(self.args)
+        
+        wait_for_workers([p_monitor, p_tensorboard, p_server])
+        
+        
+        
+        

evalscope/config.py

@@ -0,0 +1,165 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+import copy
+from dataclasses import dataclass, asdict, field
+from typing import Optional, List
+
+from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
+from evalscope.models.custom import CustomModel
+from evalscope.utils import yaml_to_dict
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+cur_path = os.path.dirname(os.path.abspath(__file__))
+
+registry_tasks = {
+    'arc': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/arc.yaml')),
+    'gsm8k': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/gsm8k.yaml')),
+    'mmlu': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu.yaml')),
+    'cmmlu': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/cmmlu.yaml')),
+    'ceval': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval.yaml')),
+    'bbh': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh.yaml')),
+    'general_qa': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/general_qa.yaml')),
+
+    # 'bbh_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh_mini.yaml')),
+    # 'mmlu_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu_mini.yaml')),
+    # 'ceval_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval_mini.yaml')),
+
+}
+
+
+@dataclass
+class TaskConfig:
+    model_args: Optional[dict] = field(default_factory=dict)
+    generation_config: Optional[dict] = field(default_factory=dict)
+    dataset_args: Optional[dict] = field(default_factory=dict)
+    dry_run: bool = False
+    model: CustomModel = None
+    eval_type: str = 'custom'
+    datasets: list = field(default_factory=list)
+    work_dir: str = DEFAULT_ROOT_CACHE_DIR
+    outputs: str = None
+    mem_cache: bool = False
+    use_cache: bool = True
+    stage: str = 'all'      # `all` or `infer` or `review`
+    dataset_hub: str = 'ModelScope'
+    dataset_dir: str = DEFAULT_ROOT_CACHE_DIR
+    limit: int = None
+    eval_backend: str = 'Native'
+    eval_config: dict = field(default_factory=dict)
+
+    # def __post_init__(self):
+    #     self.registry_tasks = {
+    #         'arc': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/arc.yaml')),
+    #         'gsm8k': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/gsm8k.yaml')),
+    #         'mmlu': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu.yaml')),
+    #         'ceval': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval.yaml')),
+    #         'bbh': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh.yaml')),
+    #
+    #         'bbh_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/bbh_mini.yaml')),
+    #         'mmlu_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/mmlu_mini.yaml')),
+    #         'ceval_mini': yaml_to_dict(os.path.join(cur_path, 'registry/tasks/ceval_mini.yaml')),
+    #
+    #     }
+
+    @staticmethod
+    def registry(name: str, data_pattern: str, dataset_dir: str = None, subset_list: list = None) -> None:
+        """
+        Register a new task (dataset) for evaluation.
+
+        Args:
+            name: str, the dataset name.
+            data_pattern: str, the data pattern for the task.
+                    e.g. `mmlu`, `ceval`, `gsm8k`, ...
+                    refer to task_config.list() for all available datasets.
+            dataset_dir: str, the directory to store multiple datasets files. e.g. /path/to/data, 
+                then your specific custom dataset directory will be /path/to/data/{name}
+            subset_list: list, the subset list for the dataset.
+                e.g. ['middle_school_politics', 'operating_system']
+                refer to the mmlu for example.  https://github.com/hendrycks/test/blob/master/categories.py
+        """
+        available_datasets = list(registry_tasks.keys())
+        if data_pattern not in available_datasets:
+            logger.error(f'No dataset found in available datasets: {available_datasets}, got data_pattern: {data_pattern}')
+            return
+
+        # Reuse the existing task config and update the datasets
+        pattern_config = registry_tasks.get(data_pattern)
+
+        custom_config = copy.deepcopy(pattern_config)
+        custom_config.update({'datasets': [data_pattern]})
+        custom_config.update({'dataset_hub': 'Local'})     # TODO: to support `ModelScope`
+        if 'dataset_args' in custom_config:
+            if data_pattern not in custom_config:
+                custom_config['dataset_args'].update({data_pattern: {}})
+        else:
+            custom_config.update({'dataset_args': {data_pattern: {}}})
+
+        if dataset_dir is not None:
+            custom_config['dataset_args'][data_pattern].update({'local_path': dataset_dir})
+
+        if subset_list is not None:
+            # custom_config['dataset_args'].get(data_pattern, {}).update({'subset_list': subset_list})
+            custom_config['dataset_args'][data_pattern].update({'subset_list': subset_list})
+
+        registry_tasks.update({name: custom_config})
+        logger.info(f'** Registered task: {name} with data pattern: {data_pattern}')
+
+    def to_dict(self):
+        # Note: to avoid serialization error for some model instance
+        _tmp_model = copy.copy(self.model)
+        self.model = None
+        res_dict = asdict(self)
+        res_dict.update({'model': _tmp_model})
+        self.model = _tmp_model
+
+        return res_dict
+
+    @staticmethod
+    def load(custom_model: CustomModel, tasks: List[str]) -> List['TaskConfig']:
+        res_list = []
+        for task_name in tasks:
+            task: dict = registry_tasks.get(task_name, None)
+            if task is None:
+                logger.error(f'No task found in tasks: {list(registry_tasks.keys())}, got task_name: {task_name}')
+                continue
+
+            res = TaskConfig(**task)
+            res.model = custom_model
+            if res.outputs is None:
+                res.outputs = os.path.join(res.work_dir,
+                                           'outputs',
+                                           f"eval_{'-'.join(tasks)}_{res.model.config['model_id']}_{res.model_args.get('revision', 'default')}")
+            res_list.append(res)
+
+        return res_list
+
+    @staticmethod
+    def list():
+        return list(registry_tasks.keys())
+
+
+class TempModel(CustomModel):
+
+    def __init__(self, config: dict):
+        super().__init__(config=config)
+
+    def predict(self, prompts: str, **kwargs):
+        return [item + ': response' for item in prompts]
+
+
+if __name__ == '__main__':
+    model = TempModel(config={'model_id': 'test-swift-dummy-model'})
+    task_config = TaskConfig()
+
+    # Register a new task
+    TaskConfig.registry(name='arc_swift', data_pattern='arc', dataset_dir='/path/to/swift_custom_work')
+
+    import json
+    swift_eval_task: List[TaskConfig] = TaskConfig.load(custom_model=model, tasks=['gsm8k', 'arc', 'arc_swift'])
+    for item in swift_eval_task:
+        print(item.to_dict())
+        print()
+

evalscope/constants.py

@@ -0,0 +1,150 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+from enum import Enum
+
+DEFAULT_ROOT_CACHE_DIR = '~/.cache/evalscope'
+
+
+class DumpMode:
+    OVERWRITE = 'overwrite'
+    APPEND = 'append'
+
+
+class MetricsConstant:
+    EPSILON = float(1e-6)
+    INVALID_VALUE = -9999999
+    ROUGE_KEYS = [
+        'rouge-1-r',
+        'rouge-1-p',
+        'rouge-1-f',
+        'rouge-2-r',
+        'rouge-2-p',
+        'rouge-2-f',
+        'rouge-l-r',
+        'rouge-l-p',
+        'rouge-l-f',
+    ]
+
+
+class MetricMembers(Enum):
+
+    # Math accuracy metric
+    MATH_ACCURACY = 'math_accuracy'
+
+    # Code pass@k metric
+    CODE_PASS_K = 'code_pass_k'
+
+    # Code rouge metric
+    ROUGE = 'rouge'
+
+    # ELO rating system for pairwise comparison
+    ELO = 'elo'
+
+    # Pairwise comparison win/lose and tie(optional)
+    PAIRWISE = 'pairwise'
+
+    # Rating score for single model
+    SCORE = 'score'
+
+
+class ArenaWinner:
+
+    MODEL_A = 'model_a'
+
+    MODEL_B = 'model_b'
+
+    TIE = 'tie'
+
+    TIE_BOTH_BAD = 'tie_both_bad'
+
+    UNKNOWN = 'unknown'
+
+
+class ArenaMode:
+    SINGLE = 'single'
+    PAIRWISE = 'pairwise'
+    PAIRWISE_BASELINE = 'pairwise_baseline'
+
+
+class OutputsStructure:
+
+    LOGS_DIR = 'logs_dir'
+
+    PREDICTIONS_DIR = 'predictions_dir'
+
+    REVIEWS_DIR = 'reviews_dir'
+
+    REPORTS_DIR = 'reports_dir'
+
+    CONFIGS_DIR = 'configs_dir'
+
+
+class AnswerKeys:
+
+    ANSWER_ID = 'answer_id'
+
+    RAW_INPUT = 'raw_input'
+
+    ORIGIN_PROMPT = 'origin_prompt'
+
+    MODEL_SPEC = 'model_spec'
+
+    SUBSET_NAME = 'subset_name'
+
+    CHOICES = 'choices'
+
+
+class ReviewKeys:
+
+    REVIEW_ID = 'review_id'
+
+    REVIEWED = 'reviewed'
+
+    REVIEWER_SPEC = 'reviewer_spec'
+
+    REVIEW_TIME = 'review_time'
+
+    MESSAGE = 'message'
+
+    CONTENT = 'content'
+
+    GOLD = 'gold'
+
+    PRED = 'pred'
+
+    RESULT = 'result'
+
+    REVIEW = 'review'
+
+
+class EvalConfigKeys:
+    CLASS_REF = 'ref'
+    CLASS_ARGS = 'args'
+    ENABLE = 'enable'
+    POSITION_BIAS_MITIGATION = 'position_bias_mitigation'
+    RANDOM_SEED = 'random_seed'
+    FN_COMPLETION_PARSER = 'fn_completion_parser'
+    COMPLETION_PARSER_KWARGS = 'completion_parser_kwargs'
+    OUTPUT_FILE = 'output_file'
+    MODEL_ID_OR_PATH = 'model_id_or_path'
+    MODEL_REVISION = 'revision'
+    GENERATION_CONFIG = 'generation_config'
+    PRECISION = 'precision'
+    TEMPLATE_TYPE = 'template_type'
+
+
+class FnCompletionParser:
+    LMSYS_PARSER: str = 'lmsys_parser'
+    RANKING_PARSER: str = 'ranking_parser'
+
+
+class PositionBiasMitigation:
+    NONE = 'none'
+    RANDOMIZE_ORDER = 'randomize_order'
+    SWAP_POSITION = 'swap_position'
+
+
+class EvalStage:
+    # Enums: `all`, `infer`, `review`
+    ALL = 'all'
+    INFER = 'infer'
+    REVIEW = 'review'

evalscope/evaluator/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.evaluator.evaluator import Evaluator