evalscope 0.5.0__py3-none-any.whl

evalscope/run_arena.py

@@ -0,0 +1,204 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# flake8: noqa
+
+import argparse
+import os
+from pathlib import Path
+import torch
+from tqdm import tqdm
+
+from evalscope.constants import EvalConfigKeys
+from evalscope.evaluator.rating_eval import RatingEvaluate
+from evalscope.models.model_adapter import ChatGenerationModelAdapter
+from evalscope.utils import get_obj_from_cfg, yaml_to_dict, jsonl_to_list, dump_jsonl_data
+from evalscope.utils.logger import get_logger
+from modelscope.utils.hf_util import GenerationConfig
+
+logger = get_logger()
+
+WORK_DIR = Path(__file__).absolute().parent
+
+
+class ArenaWorkflow:
+
+    def __init__(self, cfg_file: str, **kwargs):
+
+        self.cfg_dict = yaml_to_dict(os.path.join(WORK_DIR, cfg_file))
+        logger.info(f'**Arena Config: {self.cfg_dict}')
+
+        self.question_file: str = os.path.join(WORK_DIR, self.cfg_dict.get('question_file'))
+        self.answers_gen: dict = self.cfg_dict.get('answers_gen', {})
+        self.reviews_gen: dict = self.cfg_dict.get('reviews_gen', {})
+        self.reviewer_cfg: dict = ArenaWorkflow._get_obj_from_cfg(self.reviews_gen.get('reviewer', {}))
+
+        self.prompt_file = os.path.join(WORK_DIR, self.reviews_gen.get('prompt_file'))
+        self.review_file = os.path.join(WORK_DIR, self.reviews_gen.get('review_file'))
+
+        self.rating_gen: dict = self.cfg_dict.get('rating_gen', {})
+        self.report_file: str = os.path.join(WORK_DIR, self.rating_gen.get('report_file'))
+
+    @staticmethod
+    def _get_obj_from_cfg(obj_cfg: dict):
+        cls_ref = obj_cfg.get(EvalConfigKeys.CLASS_REF, None)
+        if not cls_ref:
+            logger.warning(
+                f'Class reference is not specified in config: {obj_cfg}')
+            return obj_cfg
+
+        cls = get_obj_from_cfg(cls_ref)
+        obj_cfg[EvalConfigKeys.CLASS_REF] = cls
+
+        return obj_cfg
+
+    def _predict_answers(self,
+                         model_id_or_path: str,
+                         model_revision: str,
+                         precision: torch.dtype,
+                         generation_config: GenerationConfig,
+                         template_type: str) -> list:
+
+        # TODO: multi-task to be supported
+        model_adapter = ChatGenerationModelAdapter(model_id=model_id_or_path,
+                                                   model_revision=model_revision,
+                                                   torch_dtype=precision,
+                                                   generation_config=generation_config,
+                                                   template_type=template_type)
+        res_list = []
+        questions_list = jsonl_to_list(self.question_file)
+        for data_d in tqdm(questions_list, total=len(questions_list), desc=f'Predicting(answers):'):
+            # {"question_id": 1, "text": "How can I improve my time management skills?", "category": "generic"}
+            text = data_d.get('text', None)
+            if not text:
+                logger.warning(f'Invalid question: {data_d}')
+                continue
+            prompt = f'Question: {text}\n\nAnswer:'
+            inputs = {'data': [prompt]}
+            res_d: dict = model_adapter.predict(inputs=inputs)
+            ans_text: str = res_d['choices'][0]['message']['content']
+
+            ans = {
+                'question_id': data_d['question_id'],
+                'text': data_d['text'],
+                'category': data_d['category'],
+                'model_id': model_id_or_path,
+                'metadata': {},
+                'answer': ans_text,
+            }
+            res_list.append(ans)
+
+        return res_list
+
+    def get_answers(self):
+        for model_name, cfg_d in self.answers_gen.items():
+            enable = cfg_d.get(EvalConfigKeys.ENABLE, True)
+            if not enable:
+                logger.warning(
+                    f'Skip model {model_name} because it is not enabled.')
+                continue
+
+            model_id_or_path = cfg_d.get(EvalConfigKeys.MODEL_ID_OR_PATH)
+            model_revision = cfg_d.get(EvalConfigKeys.MODEL_REVISION, None)
+            precision = cfg_d.get(EvalConfigKeys.PRECISION, torch.float16)
+            precision = eval(precision) if isinstance(precision, str) else precision
+            generation_config = cfg_d.get(EvalConfigKeys.GENERATION_CONFIG, {})
+            generation_config = GenerationConfig(**generation_config)
+            ans_output_file = os.path.join(WORK_DIR, cfg_d.get(EvalConfigKeys.OUTPUT_FILE))
+            template_type = cfg_d.get(EvalConfigKeys.TEMPLATE_TYPE)
+
+            answers_list = self._predict_answers(model_id_or_path=model_id_or_path,
+                                                 model_revision=model_revision,
+                                                 precision=precision,
+                                                 generation_config=generation_config,
+                                                 template_type=template_type)
+
+            dump_jsonl_data(answers_list, ans_output_file)
+            logger.info(f'Answers generated by model {model_name} and saved to {ans_output_file}')
+
+    def get_reviews(self, dry_run: bool = False):
+        enable = self.reviews_gen.get(EvalConfigKeys.ENABLE, True)
+        if enable:
+            reviewer_cls = self.reviewer_cfg.get(EvalConfigKeys.CLASS_REF)
+            if not reviewer_cls:
+                logger.warning('Skip reviews generation because class reference is not specified.')
+                return
+            reviewer_args = self.reviewer_cfg.get(EvalConfigKeys.CLASS_ARGS, {})
+            target_answers = self.reviews_gen.get('target_answers')
+            if target_answers is None:
+                # Get all answers from answers_gen config if target_answers is None
+                target_answers = [item[EvalConfigKeys.OUTPUT_FILE] for item in self.answers_gen.values()]
+            target_answers = [os.path.join(WORK_DIR, item) for item in target_answers]
+            target_answers = [file_path for file_path in target_answers if os.path.exists(file_path)]
+
+            baseline_file = self.reviews_gen.get('baseline_file', None)
+            if baseline_file:
+                baseline_file = os.path.join(WORK_DIR, baseline_file)
+
+            reference_file = self.reviews_gen.get('reference_file', None)
+            if reference_file:
+                reference_file = os.path.join(WORK_DIR, reference_file)
+
+            cache_file = self.reviews_gen.get('cache_file', None)
+            if cache_file:
+                cache_file = os.path.join(WORK_DIR, cache_file)
+
+            input_kwargs = dict(
+                prompt_file=self.prompt_file,
+                answer_file_list=target_answers,
+                review_result_file=self.review_file,
+                baseline_file=baseline_file,
+                reference_file=reference_file,
+                reviewer_args=reviewer_args,
+                cache_file=cache_file)
+
+            reviewer_obj = reviewer_cls(**input_kwargs)
+            reviewer_obj.run(dry_run=dry_run)
+            logger.info(f'Reviews with generated by reviewer and saved to {self.review_file}')
+
+        else:
+            logger.warning('Skip reviews generation because it is not enabled.')
+
+    def get_rating_results(self):
+        enable = self.rating_gen.get(EvalConfigKeys.ENABLE, True)
+        if enable:
+            report_file = os.path.join(WORK_DIR, self.rating_gen.get('report_file'))
+            metrics = self.rating_gen.get('metrics', ['elo'])
+            baseline_model = self.rating_gen.get(
+                'baseline_model') if metrics[0] == 'pairwise' else None
+            ae = RatingEvaluate(metrics=metrics, baseline_model=baseline_model)
+            res_list = ae.run(self.review_file)
+            rating_df = res_list[0]
+            logger.info(f'Rating results:\n{rating_df.to_csv()}')
+            rating_df.to_csv(report_file, index=True)
+            logger.info(f'Rating results are saved to {report_file}')
+        else:
+            logger.warning('Skip rating because it is not enabled.')
+
+    def run(self, dry_run: bool = False):
+
+        # Get all answers
+        self.get_answers()
+
+        # Get all reviews
+        self.get_reviews(dry_run=dry_run)
+
+        # Get rating results
+        self.get_rating_results()
+
+        logger.info('*** Arena workflow is finished. ***')
+
+
+def main():
+
+    # Usage: python evalscope/run_arena.py -c /path/to/xxx_cfg_arena.yaml
+
+    parser = argparse.ArgumentParser(description='LLMs evaluations with arena mode.')
+    parser.add_argument('-c', '--cfg-file', required=True)
+    parser.add_argument('--dry-run', action='store_true', default=False)
+    args = parser.parse_args()
+
+    arena_workflow = ArenaWorkflow(cfg_file=args.cfg_file)
+    arena_workflow.run(dry_run=args.dry_run)
+
+
+if __name__ == '__main__':
+    main()

evalscope/run_ms.py

@@ -0,0 +1,140 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# flake8: noqa
+
+import argparse
+import torch
+
+from evalscope.benchmarks.ceval import DATASET_ID as CEVAL_EXAM
+from evalscope.benchmarks.mmlu import DATASET_ID as MMLU
+from evalscope.benchmarks.hellaswag import DATASET_ID as HELLA_SWAG
+from evalscope.benchmarks.arc import DATASET_ID as ARC
+from evalscope.benchmarks.truthful_qa import DATASET_ID as TRUTHFUL_QA
+from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
+from evalscope.evaluator import Evaluator
+from evalscope.models.model_adapter import MultiChoiceModelAdapter, ContinuationLogitsModelAdapter
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+# TODO: add more precision
+MODEL_PRECISION_MAP = {'fp16': torch.float16, 'fp32': torch.float32, 'bf16': torch.bfloat16}
+
+"""
+Run evaluation process for ModelScope Leaderboard.
+"""
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Run evaluation on a model')
+
+    parser.add_argument('--model', help='Model id from modelscope or huggingface.', required=True)
+    parser.add_argument('--revision', help='Model revision.', required=False, default=None)
+    parser.add_argument('--precision', help='Model precision.', default='bf16')
+    parser.add_argument('--work-dir', help='root work cache dir.', default=None)
+    parser.add_argument('--outputs-dir', help='Outputs dir.', default='outputs')
+    parser.add_argument('--datasets-dir', help='Datasets dir.', default=DEFAULT_ROOT_CACHE_DIR)
+    parser.add_argument('--device-map', help='device map.', default='auto')
+    parser.add_argument('--max-eval-size', type=int, help='Max evaluation samples num for each subset', default=None)
+    parser.add_argument('--dataset-id', help='Dataset id on modelscope', required=False, default=None)
+
+    parser.add_argument('--debug',
+                        help='Debug mode, will print information for debugging.',
+                        action='store_true',
+                        default=False)
+    parser.add_argument('--dry-run',
+                        help='Dry run in single processing mode.',
+                        action='store_true',
+                        default=False)
+    parser.add_argument('--mem-cache',
+                        help='To use memory cache or not.',
+                        action='store_true',
+                        default=False)
+
+    args = parser.parse_args()
+
+    return args
+
+
+def main():
+    args = parse_args()
+    logger.info(args)
+
+    # Customize your target datasets here
+    all_benchmarks = [CEVAL_EXAM, MMLU, ARC, HELLA_SWAG, TRUTHFUL_QA]
+
+    dataset_id = args.dataset_id
+    if dataset_id is None:
+        datasets = all_benchmarks
+    elif dataset_id in all_benchmarks:
+        datasets = [dataset_id]
+    else:
+        raise ValueError(f'Unknown dataset: {dataset_id}, Supported datasets: {all_benchmarks}')
+
+    # Get model instance
+    if args.dry_run:
+        from evalscope.models.dummy_chat_model import DummyChatModel
+        model_adapter = DummyChatModel(model_cfg=dict())        # TODO
+        model_id: str = 'dummy'
+        model_revision: str = 'v1.0.0'
+        model_precision = MODEL_PRECISION_MAP.get(args.precision, torch.bfloat16)
+    else:
+        model_id: str = args.model
+        model_revision: str = args.revision
+        model_precision = MODEL_PRECISION_MAP.get(args.precision, torch.bfloat16)
+
+        model_adapter = MultiChoiceModelAdapter(model_id=model_id,
+                                                device_map=args.device_map,
+                                                torch_dtype=model_precision,
+                                                model_revision=model_revision,)
+
+    # Evaluate on each dataset
+    for dataset_name in datasets:
+        if dataset_name == CEVAL_EXAM:
+            from evalscope.benchmarks.ceval import CEVALAdapter
+            data_adapter = CEVALAdapter()
+        elif dataset_name == MMLU:
+            from evalscope.benchmarks.mmlu import MMLUAdapter
+            data_adapter = MMLUAdapter()
+        elif dataset_name == ARC:
+            from evalscope.benchmarks.arc import ARCAdapter
+            data_adapter = ARCAdapter()
+        elif dataset_name == HELLA_SWAG:
+            # Note: HellaSwag should run few-shot eval
+            from evalscope.benchmarks.hellaswag import HellaSwagAdapter
+            data_adapter = HellaSwagAdapter()
+        elif dataset_name == TRUTHFUL_QA:
+            from evalscope.benchmarks.truthful_qa import TruthfulQaAdapter
+            data_adapter = TruthfulQaAdapter()
+
+        # TODO: add more datasets here
+        else:
+            raise ValueError(f'Unknown dataset: {dataset_name}')
+
+        # TODO: add mapping
+        if dataset_name in {TRUTHFUL_QA, HELLA_SWAG} and not args.dry_run:
+            model_adapter = ContinuationLogitsModelAdapter(model_id=model_id,
+                                                           device_map=args.device_map,
+                                                           torch_dtype=model_precision,
+                                                           model_revision=model_revision, )
+
+        root_work_dir = args.work_dir if args.work_dir is not None else DEFAULT_ROOT_CACHE_DIR
+        evaluator = Evaluator(dataset_name_or_path=dataset_name,
+                              subset_list=None,
+                              data_adapter=data_adapter,
+                              model_adapter=model_adapter,
+                              use_cache=args.mem_cache,
+                              root_cache_dir=root_work_dir,
+                              outputs_dir=args.outputs_dir,
+                              is_custom_outputs_dir=True,
+                              datasets_dir=args.datasets_dir, )
+
+        infer_cfg = dict(max_length=2048, limit=args.max_eval_size)
+        evaluator.eval(infer_cfg=infer_cfg, debug=args.debug)
+
+
+if __name__ == '__main__':
+    main()
+
+    # Usage:
+    # python evalscope/run_ms.py --model ZhipuAI/chatglm2-6b --precision fp16 --dry-run --dataset-id modelscope/mmlu --limit 10
+

evalscope/summarizer.py

@@ -0,0 +1,144 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import json
+import os
+import glob
+from typing import List, Union
+
+from evalscope.config import TaskConfig
+from evalscope.constants import OutputsStructure
+from evalscope.tools.combine_reports import gen_table
+from evalscope.utils import process_outputs_structure, yaml_to_dict, EvalBackend, json_to_dict, get_latest_folder_path, \
+    csv_to_list
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+class Summarizer:
+
+    @staticmethod
+    def get_report(outputs_dir: str) -> List[dict]:
+        res_list: list = []
+
+        outputs_structure: dict = process_outputs_structure(outputs_dir, is_make=False)
+        reports_dir: str = outputs_structure.get(OutputsStructure.REPORTS_DIR)
+        if reports_dir is None:
+            raise ValueError(f'No reports directory in {outputs_dir}')
+
+        report_files: list = glob.glob(os.path.join(reports_dir, '*.json'))
+        for report_file in report_files:
+            with open(report_file, 'r') as f:
+                res_list.append(json.load(f))
+
+        report_table: str = gen_table([reports_dir])
+        logger.info(f'*** Report table ***\n{report_table}')
+
+        return res_list
+
+    @staticmethod
+    def get_report_from_cfg(task_cfg: Union[str, List[str], TaskConfig, List[TaskConfig], dict]) -> List[dict]:
+        """
+        Get report from cfg file.
+
+        Args:
+            task_cfg: task cfg file path. refer to evalscope/tasks/eval_qwen-7b-chat_v100.yaml
+
+        Returns:
+            list: list of report dict.
+            A report dict is overall report on a benchmark for specific model.
+        """
+        final_res_list: List[dict] = []
+        candidate_task_cfgs: List[dict] = []
+
+        if isinstance(task_cfg, dict):
+            candidate_task_cfgs = [task_cfg]
+        elif isinstance(task_cfg, str):
+            task_cfg: dict = yaml_to_dict(task_cfg)
+            candidate_task_cfgs = [task_cfg]
+        elif isinstance(task_cfg, TaskConfig):
+            task_cfg: dict = task_cfg.to_dict()
+            candidate_task_cfgs = [task_cfg]
+        elif isinstance(task_cfg, list):
+            for task_cfg_item in task_cfg:
+                if isinstance(task_cfg_item, str):
+                    task_cfg_item: dict = yaml_to_dict(task_cfg_item)
+                elif isinstance(task_cfg_item, TaskConfig):
+                    task_cfg_item: dict = task_cfg_item.to_dict()
+                candidate_task_cfgs.append(task_cfg_item)
+        else:
+            raise ValueError(f'Invalid task_cfg: {task_cfg}')
+
+        for candidate_task in candidate_task_cfgs:
+            logger.info(f'**Loading task cfg for summarizer: {candidate_task}')
+            eval_backend = candidate_task.get('eval_backend') or EvalBackend.NATIVE.value
+
+            if eval_backend == EvalBackend.NATIVE.value:
+                outputs_dir: str = candidate_task.get('outputs')
+                outputs_dir: str = os.path.expanduser(outputs_dir)
+                if outputs_dir is None:
+                    raise ValueError(f'No outputs_dir in {task_cfg}')
+                res_list: list = Summarizer.get_report(outputs_dir=outputs_dir)
+                final_res_list.extend(res_list)
+
+            elif eval_backend == EvalBackend.OPEN_COMPASS.value:
+                eval_config = Summarizer.parse_eval_config(candidate_task)
+
+                work_dir = eval_config.get('work_dir') or 'outputs/default'
+                if not os.path.exists(work_dir):
+                    raise ValueError(f'work_dir {work_dir} does not exist.')
+
+                res_folder_path = get_latest_folder_path(work_dir=work_dir)
+                summary_files = glob.glob(os.path.join(res_folder_path, 'summary', '*.csv'))
+                if len(summary_files) == 0:
+                    raise ValueError(f'No summary files in {res_folder_path}')
+
+                summary_file_path = summary_files[0]
+                # Example: [{'dataset': 'gsm8k', 'version': '1d7fe4', 'metric': 'accuracy', 'mode': 'gen', 'qwen-7b-chat': '53.98'}
+                summary_res: List[dict] = csv_to_list(file_path=summary_file_path)
+                final_res_list.extend(summary_res)
+            elif eval_backend == EvalBackend.VLM_EVAL_KIT.value:
+                eval_config = Summarizer.parse_eval_config(candidate_task)
+
+                work_dir = eval_config.get('work_dir') or 'outputs/default'
+                if not os.path.exists(work_dir):
+                    raise ValueError(f'work_dir {work_dir} does not exist.')
+                
+                # TODO: parse summary files: acc.csv, score.csv, score.json for different models
+                for model in eval_config['model']:
+                    if model['name'] == 'CustomAPIModel':
+                        model_name = model['type']
+                    else:
+                        model_name = model['name']
+                    summary_files = glob.glob(os.path.join(work_dir, model_name, '*.csv'))
+                    for summary_file_path in summary_files:
+                        summary_res: dict = csv_to_list(file_path=summary_file_path)[0]
+                        file_name = os.path.basename(summary_file_path).split('.')[0]
+                        final_res_list.append({file_name: summary_res})
+                
+            elif eval_backend == EvalBackend.THIRD_PARTY.value:
+                raise ValueError(f'*** The summarizer for Third party evaluation backend is not supported yet ***')
+            else:
+                raise ValueError(f'Invalid eval_backend: {eval_backend}')
+
+        return final_res_list
+
+    @staticmethod
+    def parse_eval_config(candidate_task):
+        eval_config: Union[str, dict] = candidate_task.get('eval_config')
+        assert eval_config is not None, 'Please provide eval_config for specific evaluation backend.'
+
+        if isinstance(eval_config, str):
+            if eval_config.endswith('.yaml'):
+                eval_config: dict = yaml_to_dict(eval_config)
+            elif eval_config.endswith('.json'):
+                eval_config: dict = json_to_dict(eval_config)
+            else:
+                raise ValueError(f'Invalid eval_config: {eval_config}')
+        return eval_config
+
+
+if __name__ == '__main__':
+    cfg_file = 'registry/tasks/eval_qwen-7b-chat_v100.yaml'
+    report_list = Summarizer.get_report_from_cfg(cfg_file)
+
+    print(report_list)

evalscope/third_party/__init__.py

@@ -0,0 +1 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.

evalscope/third_party/toolbench_static/__init__.py

@@ -0,0 +1,3 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.third_party.toolbench_static.toolbench_static import run_task