evalscope 0.5.0__py3-none-any.whl

evalscope/tools/combine_reports.py

@@ -0,0 +1,140 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import os
+import json
+import glob
+from tabulate import tabulate
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+"""
+Combine and generate table for reports of LLMs.
+"""
+
+
+def get_report(report_file: str):
+    data_d: dict = json.load(open(report_file, 'r'))
+    dataset_name = data_d['name']
+    score = data_d['score']     # float or dict
+    score_d = {}
+    if isinstance(score, dict):
+        # score_d = dict([(k, round(v, 4) * 100) for k, v in score.items()])
+        score_d = score
+    elif isinstance(score, float):
+        # score_d['acc'] = round(score, 4) * 100
+        score_d['acc'] = score
+    else:
+        raise ValueError(f'Unknown score type: {type(score)}')
+    # score_str = '\n'.join([str(v) + ' (' + k + ')' for k, v in score_d.items()])
+    score_str = '\n'.join(['(' + dataset_name + '/' + k + ') ' + str(v) for k, v in score_d.items()])
+
+    return {'dataset_name': dataset_name, 'score': score_str}
+
+
+def get_model_reports(model_report_dir: str):
+    model_report_dir = os.path.normpath(model_report_dir)
+    model_report_dir = model_report_dir.rstrip('reports')
+    model_info = os.path.basename(os.path.normpath(model_report_dir))
+    model_name = '_'.join(model_info.split('_')[:-1][3:])
+    report_files = glob.glob(os.path.join(model_report_dir, 'reports', '*.json'))
+
+    model_reports_d = {model_name: []}
+    for file_path in report_files:
+        report_d = get_report(file_path)
+        model_reports_d[model_name].append(report_d)
+
+    return model_reports_d
+
+
+def gen_table(reports_path_list: list):
+    table_values = []
+    headers = ['Model']
+    is_headers_set = False
+
+    for report_path in reports_path_list:
+        model_reports_d = get_model_reports(report_path)
+        for model_name, report_list in model_reports_d.items():
+            # report_list: [{'dataset_name': 'CompetitionMath', 'score': '4.42 (acc)'},
+            #               {'dataset_name': 'GSM8K', 'score': '28.51 (acc)'}]
+            report_list = sorted(report_list, key=lambda x: x['dataset_name'])
+            if not is_headers_set:
+                headers.extend([x['dataset_name'] for x in report_list])
+                is_headers_set = True
+            single_row = []
+            single_row.append(model_name)
+            for single_report in report_list:
+                # e.g. '28.51 (acc)'
+                single_row.append(single_report['score'])
+            table_values.append(single_row)
+
+    report_table = tabulate(table_values, headers=headers, tablefmt='grid')
+    return report_table
+
+class ReportsRecorder:
+    COMMON_DATASET_PATH = []
+    CUSTOM_DATASET_PATH = []
+
+    def __init__(self, oss_url: str = "", endpoint: str = ""):
+        if oss_url and endpoint:
+            import oss2
+            from oss2.credentials import EnvironmentVariableCredentialsProvider
+
+            auth = oss2.ProviderAuth(EnvironmentVariableCredentialsProvider())
+            oss_url = oss_url.replace("oss://", "").split('/')
+            bucket_name = oss_url[0]
+
+            self.object_path = "/".join(oss_url[1:])
+            self.bucket = oss2.Bucket(auth, endpoint, bucket_name)
+        else:
+            self.object_path = ""
+            self.bucket = None
+
+
+    def append_path(self, report_path: str, dataset_name: str):
+        if dataset_name == "general_qa":
+            self.CUSTOM_DATASET_PATH.append(report_path)
+        else:
+            self.COMMON_DATASET_PATH.append(report_path)
+    
+    def dump_reports(self, output_dir: str):
+        result = {
+            "CommonDataset": [],
+            "CustomDataset": []
+        }
+        for line in self.COMMON_DATASET_PATH:
+            with open(line, 'r') as f:
+                report = json.load(f)
+                result['CommonDataset'].append(report)
+        for line in self.CUSTOM_DATASET_PATH:
+            with open(line, 'r') as f:
+                report = json.load(f)
+                report.update({"name": os.path.basename(line)})
+                result['CustomDataset'].append(report)
+        
+        os.makedirs(output_dir, exist_ok=True)
+        output_file_name = "metric.json"
+        output_path = os.path.join(output_dir, output_file_name)
+        with open(output_path, 'w+') as f:
+            f.write(json.dumps(result, ensure_ascii=False, indent=4))
+        
+        if self.bucket:
+            remote_path = os.path.join(self.object_path, output_file_name)
+            logger.info(f"** Upload report to oss: {remote_path}")
+            self.bucket.put_object_from_file(remote_path, output_path)
+
+if __name__ == '__main__':
+    report_dir_1 = '/to/path/20231129_020533_default_ZhipuAI_chatglm2-6b-base_none/reports'
+    report_dir_2 = '/to/path/20231129_020533_default_ZhipuAI_chatglm2-6b_none/reports'
+
+    report_table = gen_table([report_dir_1, report_dir_2])
+    print(report_table)
+
+    # ALL VALUES ONLY FOR EXAMPLE
+    # +--------------------------+-------------------+-------------+
+    # | Model                    | CompetitionMath   | GSM8K       |
+    # +==========================+===================+=============+
+    # | ZhipuAI_chatglm2-6b-base | 25.0 (acc)        | 30.50 (acc) |
+    # +--------------------------+-------------------+-------------+
+    # | ZhipuAI_chatglm2-6b      | 30.5 (acc)        | 40.50 (acc) |
+    # +--------------------------+-------------------+-------------+

evalscope/tools/gen_mmlu_subject_mapping.py

@@ -0,0 +1,90 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+# Note: refer to https://github.com/hendrycks/test/blob/master/categories.py
+
+subcategories = {
+    'abstract_algebra': ['math'],
+    'anatomy': ['health'],
+    'astronomy': ['physics'],
+    'business_ethics': ['business'],
+    'clinical_knowledge': ['health'],
+    'college_biology': ['biology'],
+    'college_chemistry': ['chemistry'],
+    'college_computer_science': ['computer science'],
+    'college_mathematics': ['math'],
+    'college_medicine': ['health'],
+    'college_physics': ['physics'],
+    'computer_security': ['computer science'],
+    'conceptual_physics': ['physics'],
+    'econometrics': ['economics'],
+    'electrical_engineering': ['engineering'],
+    'elementary_mathematics': ['math'],
+    'formal_logic': ['philosophy'],
+    'global_facts': ['other'],
+    'high_school_biology': ['biology'],
+    'high_school_chemistry': ['chemistry'],
+    'high_school_computer_science': ['computer science'],
+    'high_school_european_history': ['history'],
+    'high_school_geography': ['geography'],
+    'high_school_government_and_politics': ['politics'],
+    'high_school_macroeconomics': ['economics'],
+    'high_school_mathematics': ['math'],
+    'high_school_microeconomics': ['economics'],
+    'high_school_physics': ['physics'],
+    'high_school_psychology': ['psychology'],
+    'high_school_statistics': ['math'],
+    'high_school_us_history': ['history'],
+    'high_school_world_history': ['history'],
+    'human_aging': ['health'],
+    'human_sexuality': ['culture'],
+    'international_law': ['law'],
+    'jurisprudence': ['law'],
+    'logical_fallacies': ['philosophy'],
+    'machine_learning': ['computer science'],
+    'management': ['business'],
+    'marketing': ['business'],
+    'medical_genetics': ['health'],
+    'miscellaneous': ['other'],
+    'moral_disputes': ['philosophy'],
+    'moral_scenarios': ['philosophy'],
+    'nutrition': ['health'],
+    'philosophy': ['philosophy'],
+    'prehistory': ['history'],
+    'professional_accounting': ['other'],
+    'professional_law': ['law'],
+    'professional_medicine': ['health'],
+    'professional_psychology': ['psychology'],
+    'public_relations': ['politics'],
+    'security_studies': ['politics'],
+    'sociology': ['culture'],
+    'us_foreign_policy': ['politics'],
+    'virology': ['health'],
+    'world_religions': ['philosophy'],
+}
+
+categories = {
+    'STEM': ['physics', 'chemistry', 'biology', 'computer science', 'math', 'engineering'],
+    'Humanities': ['history', 'philosophy', 'law'],
+    'Social Science': ['politics', 'culture', 'economics', 'geography', 'psychology'],
+    'Other': ['other', 'business', 'health'],
+}
+
+
+def main():
+
+    reversed_categories = {}
+    for category, subcategory_list in categories.items():
+        for subcategory in subcategory_list:
+            reversed_categories[subcategory] = category
+
+    subject_mapping = {}
+    for subject, subcategory_list in subcategories.items():
+        category_name: str = reversed_categories[subcategory_list[0]]
+        subject_show_name: str = ' '.join([item.capitalize() for item in subject.split('_')])
+        subject_mapping[subject] = [subject_show_name, subcategory_list[0], category_name]
+
+    print(subject_mapping)
+
+
+if __name__ == '__main__':
+    main()

evalscope/tools/rewrite_eval_results.py

@@ -0,0 +1,95 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import os
+import time
+
+from evalscope.models.custom import CustomModel
+from evalscope.run import run_task
+from evalscope.constants import DEFAULT_ROOT_CACHE_DIR
+from evalscope.utils import yaml_to_dict
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+"""
+This script is used to rewrite the evaluation results without re-running the model predictions.
+"""
+
+
+class DummyCustomModel(CustomModel):
+
+    def __init__(self, config: dict, **kwargs):
+        super(DummyCustomModel, self).__init__(config=config, **kwargs)
+
+    def predict(self, prompts: str, **kwargs):
+        # ONLY FOR DUMMY IMPLEMENTATION, DO NOT EDIT OR USE IN PRODUCTION.
+
+        response = 'The answer is C. NOTE: ONLY FOR TEST'
+
+        res_d: dict = {
+            'choices': [
+                {
+                    'index': 0,
+                    'message': {
+                        # 'content': f'The answer is B. Raw prompt: {prompt}',
+                        'content': response,
+                        'role': 'assistant'
+                    }
+                }
+            ],
+            'created': time.time(),
+            'model': self.config.get('model_id'),  # should be model_id
+            'object': 'chat.completion',
+            'usage': {
+                'completion_tokens': 0,
+                'prompt_tokens': 0,
+                'total_tokens': 0
+            }
+        }
+
+        return [res_d for _ in prompts]
+
+
+def get_task_cfg(cfg_file: str, model_instance: CustomModel):
+    if cfg_file:
+        cfg_file: str = os.path.abspath(cfg_file)
+        logger.info(f'Loading task config from {cfg_file}')
+        task_cfg_d: dict = yaml_to_dict(yaml_file=cfg_file)
+        task_cfg_d.update({'model': model_instance})
+        logger.info(f'**Task config: {task_cfg_d}')
+    else:
+        # 默认config 示例
+        task_cfg_d = {
+            'model_args': {},
+            'generation_config': {},
+            'dataset_args': {},
+            'dry_run': False,
+            'model': model_instance,  # NOTE: model_id or # model_dir or model_instance(CustomModel)
+            'eval_type': 'custom',  # NOTE: `checkpoint` or `custom` or `service`
+            'datasets': ['arc'],
+            'work_dir': DEFAULT_ROOT_CACHE_DIR,
+            'outputs': './outputs/eval_swift_dummy',
+            'mem_cache': False,
+            'dataset_hub': 'ModelScope',
+            'dataset_dir': DEFAULT_ROOT_CACHE_DIR,
+            'stage': 'all',
+            'limit': 10,
+            'debug': False
+        }
+
+    return task_cfg_d
+
+
+if __name__ == '__main__':
+    # step1: 如果outputs做了迁移，需要修改outputs/eval_xxx 中的configs/task_output_config.yaml中的路径配置
+    # step2: 执行此脚本，默认使用use_cache=True，实现免推理对eval结果进行刷新
+
+    swift_model = DummyCustomModel(config={'model_id': 'swift-model-dummy'})
+
+    task_cfg_file = '/path/to/eval_your_model_results/configs/task_output_config.yaml'
+
+    task_cfg_d = yaml_to_dict(task_cfg_file)
+    task_cfg_d.update({'model': swift_model})
+
+    eval_results: dict = run_task(task_cfg=task_cfg_d)
+    print(f'** Evaluation results finished !\n')
+

evalscope/utils/__init__.py

@@ -0,0 +1,4 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from evalscope.utils.utils import *
+from evalscope.utils.task_utils import *

evalscope/utils/arena_utils.py

@@ -0,0 +1,247 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# Copyright (c) lmsys.org.
+
+import random
+from collections import OrderedDict, defaultdict
+from typing import List, Sequence, Union
+
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def compute_elo(battles,
+                col_model_a='model_a',
+                col_model_b='model_b',
+                col_win='win',
+                tie_values=['tie', 'tie (bothbad)'],
+                k=32,
+                scale=400,
+                base=10,
+                init_rating=1000):
+    rating = defaultdict(lambda: init_rating)
+
+    for rd, model_a, model_b, win in battles[[
+            col_model_a, col_model_b, col_win
+    ]].itertuples():
+        ra = rating[model_a]
+        rb = rating[model_b]
+        ea = 1 / (1 + base**((rb - ra) / scale))
+        eb = 1 / (1 + base**((ra - rb) / scale))
+        if win == col_model_a:
+            sa = 1
+        elif win == col_model_b:
+            sa = 0
+        elif win in tie_values:
+            sa = 0.5
+        else:
+            raise Exception(f'unexpected vote {win}')
+        rating[model_a] += k * (sa - ea)
+        rating[model_b] += k * (1 - sa - eb)
+
+    return rating
+
+
+def merge_ques_ans(answer_list_all,
+                   merge_key: str = 'question_id',
+                   merge_mode: str = 'inner') -> pd.DataFrame:
+    """
+    Merge question and answer list to unifiled data.
+
+    Args:
+        answer_list_all: list of answer list,
+            e.g. [ans1_list, ans2_list, ...], an ans_list is predicted answers
+            of a specific model, must contain following columns: 'question_id',
+            'text', 'category', 'model_id', 'answer'
+        merge_key: key for dataframe merging
+        merge_mode: mode for dataframe merging,
+            e.g. 'inner', 'left', 'right', 'outer'
+
+    Returns:
+        pandas DataFrame: merged dataframe, e.g. columns are
+            ['question_id', 'gpt-3.5-turbo', 'llama2-7b']
+    """
+    ans_df = pd.DataFrame()
+    for ans_list in answer_list_all:
+        ans_list = [{
+            'question_id': item['question_id'],
+            item['model_id']: item
+        } for item in ans_list]
+        if ans_df.empty:
+            ans_df = pa.Table.from_pylist(ans_list).to_pandas()
+        else:
+            ans_df = pd.merge(
+                ans_df,
+                pa.Table.from_pylist(ans_list).to_pandas(),
+                on=merge_key,
+                how=merge_mode)
+
+    return ans_df
+
+
+def get_battle_pairs(columns: List[str], baseline_idx: int = -1) -> List[tuple]:
+    """
+    Get battle pair names from columns.
+
+    Args:
+        columns: list of column names.
+
+    Returns:
+        list of battle pairs.
+
+    Example:
+        >>> columns = ['A', 'B', 'C']
+        >>> res = get_battle_pairs(columns)
+        >>> print(res)
+        >>> [('B', 'A'), ('C', 'A'), ('C', 'B')]
+
+        >>> columns = ['A', 'B', 'C']
+        >>> res = get_battle_pairs(columns, 2)
+        >>> print(res)
+        >>> [('A', 'C'), ('B', 'C')]
+    """
+    res_list = []
+
+    cols_num = len(columns)
+    if cols_num <= 0:
+        return res_list
+
+    if baseline_idx != -1:
+        n_column = columns[baseline_idx]
+        res_list = [(column, n_column) for column in columns
+                    if column != n_column]
+    else:
+        mat = np.ones((cols_num, cols_num))
+        mat_lower_tril = np.tril(mat, k=-1)
+        x_ids, y_ids = np.where(mat_lower_tril == 1)
+        res_list = [(columns[x_id], columns[y_id])
+                    for x_id, y_id in zip(x_ids, y_ids)]
+
+    return res_list
+
+
+def get_battle_pairs_origin(columns: List[str],
+                            compare_base: bool = False,
+                            swap: bool = False):  # TODO: to refactor
+    """
+    Get battle pair names from columns.
+
+    Args:
+        columns: list of column names.
+
+    Returns:
+        list of battle pairs.
+
+    Example:
+        >>> columns = ['A', 'B', 'C']
+        >>> res = get_battle_pairs(columns)
+        >>> print(res)
+        >>> [('B', 'A'), ('C', 'A'), ('C', 'B')]
+    """
+    res_list = []
+
+    cols_num = len(columns)
+    if cols_num <= 0:
+        return res_list
+
+    if not compare_base:
+        mat = np.ones((cols_num, cols_num))
+        mat_lower_tril = np.tril(mat, k=-1)
+        x_ids, y_ids = np.where(mat_lower_tril == 1)
+        res_list = [(columns[x_id], columns[y_id])
+                    for x_id, y_id in zip(x_ids, y_ids)]
+    else:
+        for column in columns[1:]:
+            res_list.append((columns[0], column))
+
+    if swap:
+        res_list.extend([(j, i) for i, j in res_list])
+    return res_list
+
+
+def shuffle_pairwise_preferences(
+        df: pd.DataFrame, arr_is_shuffle: Sequence[int]) -> pd.DataFrame:
+    """Shuffle the outputs of a pairwise preference dataframe.
+
+    Examples
+    --------
+    >>> df = pd.DataFrame([dict(instruction='2+2', output_1='3', output_2='4', preference=2),
+                           dict(instruction='2+3', output_1='5', output_2='4', preference=1)])
+    >>> print(shuffle_pairwise_preferences(df, [True, False]))
+        instruction output_1 output_2  preference
+    0         2+2        4        3           1
+    1         2+3        5        4           1
+    """
+    col_1 = df['output_1'].copy()
+    col_2 = df['output_2'].copy()
+    df['output_1'] = np.where(arr_is_shuffle, col_2, col_1)
+    df['output_2'] = np.where(arr_is_shuffle, col_1, col_2)
+
+    if 'preference' in df.columns:
+        df['preference'] = np.where(arr_is_shuffle, 3 - df['preference'],
+                                    df['preference'])
+
+    return df
+
+
+class BattlePairSelection:
+    """
+    Select battle pairs by specific strategy.
+
+    Attributes:
+        model_elo_map(dict): map of model_id--base_elo_score
+    """
+
+    DEFAULT_K = 5
+
+    def __init__(self, model_elo_map: Union[dict, OrderedDict]):
+        # Make sure model_elo_map to be ordered when compare_base is true.
+        self.model_elo_map = model_elo_map
+
+    def top_k(self,
+              k: int = DEFAULT_K,
+              compare_base: bool = False,
+              swap: bool = False) -> list:
+        if k <= 0:
+            k = self.DEFAULT_K
+        sorted_res = sorted(self.model_elo_map.items(), key=lambda x: x[1])[:k]
+        sorted_res = list(dict(sorted_res).keys())
+        return get_battle_pairs_origin(sorted_res, compare_base, swap)
+
+    def random_k(self,
+                 k: int = DEFAULT_K,
+                 compare_base: bool = False,
+                 swap: bool = False) -> list:
+        if k <= 0:
+            k = self.DEFAULT_K
+        if k > len(self.model_elo_map):
+            k = len(self.model_elo_map)
+        candidate_list = list(self.model_elo_map.items())
+        k = len(candidate_list) if k > len(candidate_list) else k
+        res = dict(random.sample(candidate_list, k=k))
+        res = list(res.keys())
+        return get_battle_pairs_origin(res, compare_base, swap)
+
+    def volatility_index(self,
+                         frac: float = 0.2,
+                         compare_base: bool = False,
+                         swap: bool = False) -> list:
+        res_list = []
+        candidate_list = get_battle_pairs_origin(
+            list(self.model_elo_map.keys()), compare_base, swap)
+        for t in candidate_list:
+            model_a = t[0]
+            model_b = t[1]
+            base_elo_a = self.model_elo_map.get(model_a)
+            base_elo_b = self.model_elo_map.get(model_b)
+
+            vol_frac = abs(base_elo_b - base_elo_a) / max(
+                base_elo_a, base_elo_b)
+            if vol_frac <= frac:
+                res_list.append(t)
+
+        return res_list

evalscope/utils/completion_parsers.py

@@ -0,0 +1,87 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+# flake8: noqa
+
+import ast
+import re
+
+
+# from . import utils as ann_utils
+from evalscope.constants import ArenaWinner
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+one_score_pattern = re.compile('\[\[(\d+\.?\d*)\]\]')
+one_score_pattern_backup = re.compile('\[(\d+\.?\d*)\]')
+
+
+# modified from: https://github.com/lm-sys/FastChat/blob/main/fastchat/eval/eval_gpt_review.py#L47
+# does not work with batched completions
+def lmsys_parser(completion, output_format):
+    if output_format == '[[rating]]':
+        match = re.search(one_score_pattern, completion)
+        if not match:
+            match = re.search(one_score_pattern_backup, completion)
+
+        if match:
+            rating = ast.literal_eval(match.groups()[0])
+        else:
+            logger.error(f'Content: {completion}\n'
+                         'You must manually fix the score.')
+            rating = -1
+
+        return rating
+    if output_format == '[[rating_a,rating_b]]':
+        try:
+            score_pair = completion.split('\n')[0]
+            score_pair = score_pair.replace(',', ' ')
+            sp = score_pair.split(' ')
+            if len(sp) == 2:
+                score_1 = float(sp[0])
+                score_2 = float(sp[1])
+                if score_1 > score_2:
+                    winner = ArenaWinner.MODEL_A
+                elif score_1 < score_2:
+                    winner = ArenaWinner.MODEL_B
+                else:
+                    if score_1 == score_1 == -1:
+                        winner = ArenaWinner.UNKNOWN
+                    winner = ArenaWinner.TIE
+                return winner, [score_1, score_2]
+            else:
+                raise Exception('Invalid score pair.')
+        except Exception as e:
+            logger.error(
+                f'{e}\nContent: {completion}\nYou must manually fix the score pair.'
+            )
+            return ArenaWinner.UNKNOWN, [-1, -1]
+    elif output_format == '[[A]]':
+        if '[[A]]' in completion:
+            winner = ArenaWinner.MODEL_A
+        elif '[[B]]' in completion:
+            winner = ArenaWinner.MODEL_B
+        elif '[[C]]' in completion:
+            winner = ArenaWinner.TIE
+        else:
+            logger.error(
+                f'\nContent: {completion}\nYou must manually fix the score.')
+            winner = ArenaWinner.UNKNOWN
+        return winner
+
+
+def ranking_parser(completion, **kwargs):
+    try:
+        if isinstance(completion, str):
+            ordered_completions = ast.literal_eval(completion)
+        else:
+            ordered_completions = completion
+
+        rank = [c for c in ordered_completions
+                if c['model'] == 'model_a'][0]['rank']
+        assert rank in [1, 2]
+
+        return ArenaWinner.MODEL_A if rank == 1 else ArenaWinner.MODEL_B
+    except Exception as e:
+        logger.error(f'{e}\nContent: {completion}\n'
+                     'You must manually fix the score pair.')
+        return ArenaWinner.UNKNOWN