evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/run_arena.py

@@ -1,202 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# flake8: noqa
-
-import argparse
-import os
-import torch
-from modelscope.utils.hf_util import GenerationConfig
-from pathlib import Path
-from tqdm import tqdm
-
-from evalscope.constants import EvalConfigKeys
-from evalscope.evaluator.rating_eval import RatingEvaluate
-from evalscope.models import ChatGenerationModelAdapter
-from evalscope.utils import get_obj_from_cfg
-from evalscope.utils.io_utils import dump_jsonl_data, jsonl_to_list, yaml_to_dict
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-WORK_DIR = Path(__file__).absolute().parent
-
-
-class ArenaWorkflow:
-
-    def __init__(self, cfg_file: str, **kwargs):
-
-        self.cfg_dict = yaml_to_dict(os.path.join(WORK_DIR, cfg_file))
-        logger.info(f'**Arena Config: {self.cfg_dict}')
-
-        self.question_file: str = os.path.join(WORK_DIR, self.cfg_dict.get('question_file'))
-        self.answers_gen: dict = self.cfg_dict.get('answers_gen', {})
-        self.reviews_gen: dict = self.cfg_dict.get('reviews_gen', {})
-        self.reviewer_cfg: dict = ArenaWorkflow._get_obj_from_cfg(self.reviews_gen.get('reviewer', {}))
-
-        self.prompt_file = os.path.join(WORK_DIR, self.reviews_gen.get('prompt_file'))
-        self.review_file = os.path.join(WORK_DIR, self.reviews_gen.get('review_file'))
-
-        self.rating_gen: dict = self.cfg_dict.get('rating_gen', {})
-        self.report_file: str = os.path.join(WORK_DIR, self.rating_gen.get('report_file'))
-
-    @staticmethod
-    def _get_obj_from_cfg(obj_cfg: dict):
-        cls_ref = obj_cfg.get(EvalConfigKeys.CLASS_REF, None)
-        if not cls_ref:
-            logger.warning(f'Class reference is not specified in config: {obj_cfg}')
-            return obj_cfg
-
-        cls = get_obj_from_cfg(cls_ref)
-        obj_cfg[EvalConfigKeys.CLASS_REF] = cls
-
-        return obj_cfg
-
-    def _predict_answers(self, model_id_or_path: str, model_revision: str, precision: torch.dtype,
-                         generation_config: GenerationConfig, template_type: str) -> list:
-
-        # TODO: multi-task to be supported
-        model_adapter = ChatGenerationModelAdapter(
-            model_id=model_id_or_path,
-            model_revision=model_revision,
-            torch_dtype=precision,
-            generation_config=generation_config,
-            template_type=template_type)
-        res_list = []
-        questions_list = jsonl_to_list(self.question_file)
-        for data_d in tqdm(questions_list, total=len(questions_list), desc=f'Predicting(answers):'):
-            # {"question_id": 1, "text": "How can I improve my time management skills?", "category": "generic"}
-            text = data_d.get('text', None)
-            if not text:
-                logger.warning(f'Invalid question: {data_d}')
-                continue
-            prompt = f'Question: {text}\n\nAnswer:'
-            inputs = {'data': [prompt]}
-            res_d: dict = model_adapter.predict(inputs=inputs)
-            ans_text: str = res_d['choices'][0]['message']['content']
-
-            ans = {
-                'question_id': data_d['question_id'],
-                'text': data_d['text'],
-                'category': data_d['category'],
-                'model_id': model_id_or_path,
-                'metadata': {},
-                'answer': ans_text,
-            }
-            res_list.append(ans)
-
-        return res_list
-
-    def get_answers(self):
-        for model_name, cfg_d in self.answers_gen.items():
-            enable = cfg_d.get(EvalConfigKeys.ENABLE, True)
-            if not enable:
-                logger.warning(f'Skip model {model_name} because it is not enabled.')
-                continue
-
-            model_id_or_path = cfg_d.get(EvalConfigKeys.MODEL_ID_OR_PATH)
-            model_revision = cfg_d.get(EvalConfigKeys.MODEL_REVISION, None)
-            precision = cfg_d.get(EvalConfigKeys.PRECISION, torch.float16)
-            precision = eval(precision) if isinstance(precision, str) else precision
-            custom_generation_config = cfg_d.get(EvalConfigKeys.GENERATION_CONFIG, {})
-            custom_generation_config = GenerationConfig(**custom_generation_config)
-            ans_output_file = os.path.join(WORK_DIR, cfg_d.get(EvalConfigKeys.OUTPUT_FILE))
-            template_type = cfg_d.get(EvalConfigKeys.TEMPLATE_TYPE)
-
-            answers_list = self._predict_answers(
-                model_id_or_path=model_id_or_path,
-                model_revision=model_revision,
-                precision=precision,
-                generation_config=custom_generation_config,
-                template_type=template_type)
-
-            os.makedirs(os.path.dirname(ans_output_file), exist_ok=True)
-            dump_jsonl_data(answers_list, ans_output_file)
-            logger.info(f'Answers generated by model {model_name} and saved to {ans_output_file}')
-
-    def get_reviews(self, dry_run: bool = False):
-        enable = self.reviews_gen.get(EvalConfigKeys.ENABLE, True)
-        if enable:
-            reviewer_cls = self.reviewer_cfg.get(EvalConfigKeys.CLASS_REF)
-            if not reviewer_cls:
-                logger.warning('Skip reviews generation because class reference is not specified.')
-                return
-            reviewer_args = self.reviewer_cfg.get(EvalConfigKeys.CLASS_ARGS, {})
-            target_answers = self.reviews_gen.get('target_answers')
-            if target_answers is None:
-                # Get all answers from answers_gen config if target_answers is None
-                target_answers = [item[EvalConfigKeys.OUTPUT_FILE] for item in self.answers_gen.values()]
-            target_answers = [os.path.join(WORK_DIR, item) for item in target_answers]
-            target_answers = [file_path for file_path in target_answers if os.path.exists(file_path)]
-
-            baseline_file = self.reviews_gen.get('baseline_file', None)
-            if baseline_file:
-                baseline_file = os.path.join(WORK_DIR, baseline_file)
-
-            reference_file = self.reviews_gen.get('reference_file', None)
-            if reference_file:
-                reference_file = os.path.join(WORK_DIR, reference_file)
-
-            cache_file = self.reviews_gen.get('cache_file', None)
-            if cache_file:
-                cache_file = os.path.join(WORK_DIR, cache_file)
-
-            input_kwargs = dict(
-                prompt_file=self.prompt_file,
-                answer_file_list=target_answers,
-                review_result_file=self.review_file,
-                baseline_file=baseline_file,
-                reference_file=reference_file,
-                reviewer_args=reviewer_args,
-                cache_file=cache_file)
-
-            reviewer_obj = reviewer_cls(**input_kwargs)
-            reviewer_obj.run(dry_run=dry_run)
-            logger.info(f'Reviews with generated by reviewer and saved to {self.review_file}')
-
-        else:
-            logger.warning('Skip reviews generation because it is not enabled.')
-
-    def get_rating_results(self):
-        enable = self.rating_gen.get(EvalConfigKeys.ENABLE, True)
-        if enable:
-            report_file = os.path.join(WORK_DIR, self.rating_gen.get('report_file'))
-            metrics = self.rating_gen.get('metrics', ['elo'])
-            baseline_model = self.rating_gen.get('baseline_model') if metrics[0] == 'pairwise' else None
-            ae = RatingEvaluate(metrics=metrics, baseline_model=baseline_model)
-            res_list = ae.run(self.review_file)
-            rating_df = res_list[0]
-            logger.info(f'Rating results:\n{rating_df.to_csv()}')
-            os.makedirs(os.path.dirname(report_file), exist_ok=True)
-            rating_df.to_csv(report_file, index=True)
-            logger.info(f'Rating results are saved to {report_file}')
-        else:
-            logger.warning('Skip rating because it is not enabled.')
-
-    def run(self, dry_run: bool = False):
-
-        # Get all answers
-        self.get_answers()
-
-        # Get all reviews
-        self.get_reviews(dry_run=dry_run)
-
-        # Get rating results
-        self.get_rating_results()
-
-        logger.info('*** Arena workflow is finished. ***')
-
-
-def main():
-
-    # Usage: python evalscope/run_arena.py -c /path/to/xxx_cfg_arena.yaml
-
-    parser = argparse.ArgumentParser(description='LLMs evaluations with arena mode.')
-    parser.add_argument('-c', '--cfg-file', required=True)
-    parser.add_argument('--dry-run', action='store_true', default=False)
-    args = parser.parse_args()
-
-    arena_workflow = ArenaWorkflow(cfg_file=args.cfg_file)
-    arena_workflow.run(dry_run=args.dry_run)
-
-
-if __name__ == '__main__':
-    main()

evalscope/utils/arena_utils.py

@@ -1,217 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# Copyright (c) lmsys.org.
-
-import numpy as np
-import pandas as pd
-import pyarrow as pa
-import random
-from collections import OrderedDict, defaultdict
-from typing import List, Sequence, Union
-
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-
-def compute_elo(battles,
-                col_model_a='model_a',
-                col_model_b='model_b',
-                col_win='win',
-                tie_values=['tie', 'tie (bothbad)'],
-                k=32,
-                scale=400,
-                base=10,
-                init_rating=1000):
-    rating = defaultdict(lambda: init_rating)
-
-    for rd, model_a, model_b, win in battles[[col_model_a, col_model_b, col_win]].itertuples():
-        ra = rating[model_a]
-        rb = rating[model_b]
-        ea = 1 / (1 + base**((rb - ra) / scale))
-        eb = 1 / (1 + base**((ra - rb) / scale))
-        if win == col_model_a:
-            sa = 1
-        elif win == col_model_b:
-            sa = 0
-        elif win in tie_values:
-            sa = 0.5
-        else:
-            raise Exception(f'unexpected vote {win}')
-        rating[model_a] += k * (sa - ea)
-        rating[model_b] += k * (1 - sa - eb)
-
-    return rating
-
-
-def merge_ques_ans(answer_list_all, merge_key: str = 'question_id', merge_mode: str = 'inner') -> pd.DataFrame:
-    """
-    Merge question and answer list to unifiled data.
-
-    Args:
-        answer_list_all: list of answer list,
-            e.g. [ans1_list, ans2_list, ...], an ans_list is predicted answers
-            of a specific model, must contain following columns: 'question_id',
-            'text', 'category', 'model_id', 'answer'
-        merge_key: key for dataframe merging
-        merge_mode: mode for dataframe merging,
-            e.g. 'inner', 'left', 'right', 'outer'
-
-    Returns:
-        pandas DataFrame: merged dataframe, e.g. columns are
-            ['question_id', 'gpt-3.5-turbo', 'llama2-7b']
-    """
-    ans_df = pd.DataFrame()
-    for ans_list in answer_list_all:
-        ans_list = [{'question_id': item['question_id'], item['model_id']: item} for item in ans_list]
-        if ans_df.empty:
-            ans_df = pa.Table.from_pylist(ans_list).to_pandas()
-        else:
-            ans_df = pd.merge(ans_df, pa.Table.from_pylist(ans_list).to_pandas(), on=merge_key, how=merge_mode)
-
-    return ans_df
-
-
-def get_battle_pairs(columns: List[str], baseline_idx: int = -1) -> List[tuple]:
-    """
-    Get battle pair names from columns.
-
-    Args:
-        columns: list of column names.
-
-    Returns:
-        list of battle pairs.
-
-    Example:
-        >>> columns = ['A', 'B', 'C']
-        >>> res = get_battle_pairs(columns)
-        >>> print(res)
-        >>> [('B', 'A'), ('C', 'A'), ('C', 'B')]
-
-        >>> columns = ['A', 'B', 'C']
-        >>> res = get_battle_pairs(columns, 2)
-        >>> print(res)
-        >>> [('A', 'C'), ('B', 'C')]
-    """
-    res_list = []
-
-    cols_num = len(columns)
-    if cols_num <= 0:
-        return res_list
-
-    if baseline_idx != -1:
-        n_column = columns[baseline_idx]
-        res_list = [(column, n_column) for column in columns if column != n_column]
-    else:
-        mat = np.ones((cols_num, cols_num))
-        mat_lower_tril = np.tril(mat, k=-1)
-        x_ids, y_ids = np.where(mat_lower_tril == 1)
-        res_list = [(columns[x_id], columns[y_id]) for x_id, y_id in zip(x_ids, y_ids)]
-
-    return res_list
-
-
-def get_battle_pairs_origin(columns: List[str], compare_base: bool = False, swap: bool = False):  # TODO: to refactor
-    """
-    Get battle pair names from columns.
-
-    Args:
-        columns: list of column names.
-
-    Returns:
-        list of battle pairs.
-
-    Example:
-        >>> columns = ['A', 'B', 'C']
-        >>> res = get_battle_pairs(columns)
-        >>> print(res)
-        >>> [('B', 'A'), ('C', 'A'), ('C', 'B')]
-    """
-    res_list = []
-
-    cols_num = len(columns)
-    if cols_num <= 0:
-        return res_list
-
-    if not compare_base:
-        mat = np.ones((cols_num, cols_num))
-        mat_lower_tril = np.tril(mat, k=-1)
-        x_ids, y_ids = np.where(mat_lower_tril == 1)
-        res_list = [(columns[x_id], columns[y_id]) for x_id, y_id in zip(x_ids, y_ids)]
-    else:
-        for column in columns[1:]:
-            res_list.append((columns[0], column))
-
-    if swap:
-        res_list.extend([(j, i) for i, j in res_list])
-    return res_list
-
-
-def shuffle_pairwise_preferences(df: pd.DataFrame, arr_is_shuffle: Sequence[int]) -> pd.DataFrame:
-    """Shuffle the outputs of a pairwise preference dataframe.
-
-    Examples
-    --------
-    >>> df = pd.DataFrame([dict(instruction='2+2', output_1='3', output_2='4', preference=2),
-                           dict(instruction='2+3', output_1='5', output_2='4', preference=1)])
-    >>> print(shuffle_pairwise_preferences(df, [True, False]))
-        instruction output_1 output_2  preference
-    0         2+2        4        3           1
-    1         2+3        5        4           1
-    """
-    col_1 = df['output_1'].copy()
-    col_2 = df['output_2'].copy()
-    df['output_1'] = np.where(arr_is_shuffle, col_2, col_1)
-    df['output_2'] = np.where(arr_is_shuffle, col_1, col_2)
-
-    if 'preference' in df.columns:
-        df['preference'] = np.where(arr_is_shuffle, 3 - df['preference'], df['preference'])
-
-    return df
-
-
-class BattlePairSelection:
-    """
-    Select battle pairs by specific strategy.
-
-    Attributes:
-        model_elo_map(dict): map of model_id--base_elo_score
-    """
-
-    DEFAULT_K = 5
-
-    def __init__(self, model_elo_map: Union[dict, OrderedDict]):
-        # Make sure model_elo_map to be ordered when compare_base is true.
-        self.model_elo_map = model_elo_map
-
-    def top_k(self, k: int = DEFAULT_K, compare_base: bool = False, swap: bool = False) -> list:
-        if k <= 0:
-            k = self.DEFAULT_K
-        sorted_res = sorted(self.model_elo_map.items(), key=lambda x: x[1])[:k]
-        sorted_res = list(dict(sorted_res).keys())
-        return get_battle_pairs_origin(sorted_res, compare_base, swap)
-
-    def random_k(self, k: int = DEFAULT_K, compare_base: bool = False, swap: bool = False) -> list:
-        if k <= 0:
-            k = self.DEFAULT_K
-        if k > len(self.model_elo_map):
-            k = len(self.model_elo_map)
-        candidate_list = list(self.model_elo_map.items())
-        k = len(candidate_list) if k > len(candidate_list) else k
-        res = dict(random.sample(candidate_list, k=k))
-        res = list(res.keys())
-        return get_battle_pairs_origin(res, compare_base, swap)
-
-    def volatility_index(self, frac: float = 0.2, compare_base: bool = False, swap: bool = False) -> list:
-        res_list = []
-        candidate_list = get_battle_pairs_origin(list(self.model_elo_map.keys()), compare_base, swap)
-        for t in candidate_list:
-            model_a = t[0]
-            model_b = t[1]
-            base_elo_a = self.model_elo_map.get(model_a)
-            base_elo_b = self.model_elo_map.get(model_b)
-
-            vol_frac = abs(base_elo_b - base_elo_a) / max(base_elo_a, base_elo_b)
-            if vol_frac <= frac:
-                res_list.append(t)
-
-        return res_list

evalscope/utils/completion_parsers.py

@@ -1,82 +0,0 @@
-# Copyright (c) Alibaba, Inc. and its affiliates.
-# flake8: noqa
-
-import ast
-import re
-
-# from . import utils as ann_utils
-from evalscope.constants import ArenaWinner
-from evalscope.utils.logger import get_logger
-
-logger = get_logger()
-
-one_score_pattern = re.compile('\[\[(\d+\.?\d*)\]\]')
-one_score_pattern_backup = re.compile('\[(\d+\.?\d*)\]')
-
-
-# modified from: https://github.com/lm-sys/FastChat/blob/main/fastchat/eval/eval_gpt_review.py#L47
-# does not work with batched completions
-def lmsys_parser(completion, output_format):
-    if output_format == '[[rating]]':
-        match = re.search(one_score_pattern, completion)
-        if not match:
-            match = re.search(one_score_pattern_backup, completion)
-
-        if match:
-            rating = ast.literal_eval(match.groups()[0])
-        else:
-            logger.error(f'Content: {completion}\n'
-                         'You must manually fix the score.')
-            rating = -1
-
-        return rating
-    if output_format == '[[rating_a,rating_b]]':
-        try:
-            score_pair = completion.split('\n')[0]
-            score_pair = score_pair.replace(',', ' ')
-            sp = score_pair.split(' ')
-            if len(sp) == 2:
-                score_1 = float(sp[0])
-                score_2 = float(sp[1])
-                if score_1 > score_2:
-                    winner = ArenaWinner.MODEL_A
-                elif score_1 < score_2:
-                    winner = ArenaWinner.MODEL_B
-                else:
-                    if score_1 == score_1 == -1:
-                        winner = ArenaWinner.UNKNOWN
-                    winner = ArenaWinner.TIE
-                return winner, [score_1, score_2]
-            else:
-                raise Exception('Invalid score pair.')
-        except Exception as e:
-            logger.error(f'{e}\nContent: {completion}\nYou must manually fix the score pair.')
-            return ArenaWinner.UNKNOWN, [-1, -1]
-    elif output_format == '[[A]]':
-        if '[[A]]' in completion:
-            winner = ArenaWinner.MODEL_A
-        elif '[[B]]' in completion:
-            winner = ArenaWinner.MODEL_B
-        elif '[[C]]' in completion:
-            winner = ArenaWinner.TIE
-        else:
-            logger.error(f'\nContent: {completion}\nYou must manually fix the score.')
-            winner = ArenaWinner.UNKNOWN
-        return winner
-
-
-def ranking_parser(completion, **kwargs):
-    try:
-        if isinstance(completion, str):
-            ordered_completions = ast.literal_eval(completion)
-        else:
-            ordered_completions = completion
-
-        rank = [c for c in ordered_completions if c['model'] == 'model_a'][0]['rank']
-        assert rank in [1, 2]
-
-        return ArenaWinner.MODEL_A if rank == 1 else ArenaWinner.MODEL_B
-    except Exception as e:
-        logger.error(f'{e}\nContent: {completion}\n'
-                     'You must manually fix the score pair.')
-        return ArenaWinner.UNKNOWN