evalscope 0.7.2__py3-none-any.whl → 0.8.1__py3-none-any.whl

evalscope/evaluator/rating_eval.py

@@ -1,24 +1,17 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from typing import List, Union
-
 import pandas as pd
 import pyarrow as pa
+from typing import List, Union
 
 from evalscope.constants import MetricMembers
 from evalscope.utils.arena_utils import compute_elo
+from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
-from evalscope.utils import jsonl_to_list
 
 logger = get_logger()
 
-DEFAULT_COLUMNS_MAPPING = {
-    'model_a': 'model_a',
-    'model_b': 'model_b',
-    'win': 'win',
-    'tstamp': 'ts',
-    'language': 'lang'
-}
+DEFAULT_COLUMNS_MAPPING = {'model_a': 'model_a', 'model_b': 'model_b', 'win': 'win', 'tstamp': 'ts', 'language': 'lang'}
 
 
 class RatingEvaluate(object):
@@ -41,10 +34,9 @@ class RatingEvaluate(object):
         elo_ratings = compute_elo(battles)
         col_model = 'Model'
         col_elo_rating = 'Elo_Rating'
-        elo_ratings_res = pd.DataFrame(
-            [[n, elo_ratings[n]] for n in elo_ratings.keys()],
-            columns=[col_model, col_elo_rating]).sort_values(
-                col_elo_rating, ascending=False).reset_index(drop=True)
+        elo_ratings_res = pd.DataFrame([[n, elo_ratings[n]] for n in elo_ratings.keys()],
+                                       columns=[col_model, col_elo_rating]).sort_values(
+                                           col_elo_rating, ascending=False).reset_index(drop=True)
         elo_ratings_res = elo_ratings_res.round({col_elo_rating: 1})
         return elo_ratings_res
 
@@ -89,23 +81,11 @@ class RatingEvaluate(object):
                 'tie': 1
             }]
         else:
-            return [{
-                'model': winner,
-                'win': 1,
-                'loss': 0,
-                'tie': 0
-            }, {
-                'model': loser,
-                'win': 0,
-                'loss': 1,
-                'tie': 0
-            }]
+            return [{'model': winner, 'win': 1, 'loss': 0, 'tie': 0}, {'model': loser, 'win': 0, 'loss': 1, 'tie': 0}]
 
     def compute_pairwise_rating(self, raw_data):
         df_all = self.preprocess(raw_data_df=raw_data)
-        model_list = (
-            df_all['model_a'].unique().tolist()
-            + df_all['model_b'].unique().tolist())
+        model_list = (df_all['model_a'].unique().tolist() + df_all['model_b'].unique().tolist())
         model_list = list(set(model_list))
 
         list_res = []
@@ -114,8 +94,7 @@ class RatingEvaluate(object):
             if self.baseline_model is not None:
                 if self.baseline_model not in [row['model_a'], row['model_b']]:
                     logger.warning(
-                        f'One of the models in the battle should be the baseline model: {self.baseline_model}'
-                    )
+                        f'One of the models in the battle should be the baseline model: {self.baseline_model}')
                     continue
             rating = self.get_single_pairwise_rating(row)
             list_res = list_res + rating
@@ -149,15 +128,15 @@ class RatingEvaluate(object):
 
         for metric in self.metrics:
 
-            if metric == MetricMembers.ELO.value:
+            if metric == MetricMembers.ELO:
                 res = self.compute_elo_rating(raw_data)
                 res_all.append(res)
 
-            elif metric == MetricMembers.PAIRWISE.value:
+            elif metric == MetricMembers.PAIRWISE:
                 res = self.compute_pairwise_rating(raw_data)
                 res_all.append(res)
 
-            elif metric == MetricMembers.SCORE.value:
+            elif metric == MetricMembers.SCORE:
                 res = self.compute_score_rating(raw_data)
                 res_all.append(res)
 

evalscope/evaluator/reviewer/auto_reviewer.py

@@ -2,6 +2,7 @@
 # flake8: noqa
 
 import os
+import pandas as pd
 import random
 import sys
 import time
@@ -9,15 +10,11 @@ from abc import ABC, abstractmethod
 from functools import partial
 from typing import Any, List
 
-import pandas as pd
-
 from evalscope.constants import ArenaMode, EvalConfigKeys, FnCompletionParser, PositionBiasMitigation
 from evalscope.models.openai_model import OpenAIModel
-from evalscope.utils import completion_parsers
-from evalscope.utils.arena_utils import (get_battle_pairs,
-                                         merge_ques_ans,
-                                         shuffle_pairwise_preferences)
-from evalscope.utils import dump_jsonl_data, jsonl_to_list, random_seeded_choice
+from evalscope.utils import completion_parsers, random_seeded_choice
+from evalscope.utils.arena_utils import get_battle_pairs, merge_ques_ans, shuffle_pairwise_preferences
+from evalscope.utils.io_utils import dump_jsonl_data, jsonl_to_list
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -33,8 +30,7 @@ class BaseReviewer(ABC):
         """
         Run pairwise battles with given models.
         """
-        raise NotImplementedError(
-            'run() method must be implemented in your subclass.')
+        raise NotImplementedError('run() method must be implemented in your subclass.')
 
 
 class AutoReviewerGpt4(BaseReviewer):
@@ -71,13 +67,9 @@ class AutoReviewerGpt4(BaseReviewer):
 
         self.review_result_file = review_result_file
         self.prompt_list = jsonl_to_list(prompt_file)
-        self.answer_list = [
-            jsonl_to_list(answer_file) for answer_file in answer_file_list
-        ]
-        self.reference_list = jsonl_to_list(
-            reference_file) if reference_file else []
-        self.cache_list = jsonl_to_list(
-            cache_file) if cache_file and os.path.isfile(cache_file) else []
+        self.answer_list = [jsonl_to_list(answer_file) for answer_file in answer_file_list]
+        self.reference_list = jsonl_to_list(reference_file) if reference_file else []
+        self.cache_list = jsonl_to_list(cache_file) if cache_file and os.path.isfile(cache_file) else []
 
         self.reviewer_args = reviewer_args if reviewer_args \
             else self._get_default_args()
@@ -88,24 +80,18 @@ class AutoReviewerGpt4(BaseReviewer):
             self.answer_list.append(jsonl_to_list(baseline_file))
             self.baseline_idx = len(self.answer_list) - 1
 
-        self.position_bias_mitigation = self.reviewer_args.pop(
-            EvalConfigKeys.POSITION_BIAS_MITIGATION,
-            PositionBiasMitigation.NONE)
+        self.position_bias_mitigation = self.reviewer_args.pop(EvalConfigKeys.POSITION_BIAS_MITIGATION,
+                                                               PositionBiasMitigation.NONE)
         if self.position_bias_mitigation == PositionBiasMitigation.RANDOMIZE_ORDER:
-            self.random_seed = self.reviewer_args.pop(
-                EvalConfigKeys.RANDOM_SEED, 123)
-
-        fn_completion_parser = self.reviewer_args.pop(
-            EvalConfigKeys.FN_COMPLETION_PARSER,
-            FnCompletionParser.LMSYS_PARSER)
-        completion_parser_kwargs = self.reviewer_args.pop(
-            EvalConfigKeys.COMPLETION_PARSER_KWARGS, {})
+            self.random_seed = self.reviewer_args.pop(EvalConfigKeys.RANDOM_SEED, 123)
+
+        fn_completion_parser = self.reviewer_args.pop(EvalConfigKeys.FN_COMPLETION_PARSER,
+                                                      FnCompletionParser.LMSYS_PARSER)
+        completion_parser_kwargs = self.reviewer_args.pop(EvalConfigKeys.COMPLETION_PARSER_KWARGS, {})
         if isinstance(fn_completion_parser, str):
-            fn_completion_parser = getattr(completion_parsers,
-                                           fn_completion_parser)
+            fn_completion_parser = getattr(completion_parsers, fn_completion_parser)
 
-        self.fn_completion_parser = partial(fn_completion_parser,
-                                            **completion_parser_kwargs)
+        self.fn_completion_parser = partial(fn_completion_parser, **completion_parser_kwargs)
         self.gpt_predictor = OpenAIModel(model_cfg=self.reviewer_args)
 
     @staticmethod
@@ -133,45 +119,35 @@ class AutoReviewerGpt4(BaseReviewer):
         # Default to general category (idx 0)
         target_prompt_dict = prompts_list[0]
         for item in prompts_list:
-            is_category_match = category in item['category'] if isinstance(
-                item['category'], list) else item['category'] == category
+            is_category_match = category in item['category'] if isinstance(item['category'],
+                                                                           list) else item['category'] == category
             is_type_match = item.get('type', ArenaMode.PAIRWISE) == type
             if is_category_match and is_type_match:
                 target_prompt_dict = item
                 break
-            elif is_type_match and target_prompt_dict.get('type',
-                                                          ArenaMode.PAIRWISE) != type:
+            elif is_type_match and target_prompt_dict.get('type', ArenaMode.PAIRWISE) != type:
                 target_prompt_dict = item  # fallback to type match
 
         sys_prompt = target_prompt_dict['system_prompt']
         prompt_template = target_prompt_dict['prompt_template']
         defaults = target_prompt_dict.get('defaults', dict({}))
-        output_format = target_prompt_dict.get('output_format',
-                                               '[[rating_a,rating_b]]')
+        output_format = target_prompt_dict.get('output_format', '[[rating_a,rating_b]]')
 
         if type == ArenaMode.SINGLE:
-            user_prompt = prompt_template.format(
-                question=ques, answer=ans1, ref_answer_1=ans_ref, **defaults)
+            user_prompt = prompt_template.format(question=ques, answer=ans1, ref_answer_1=ans_ref, **defaults)
         else:
             user_prompt = prompt_template.format(
-                question=ques,
-                answer_a=ans1,
-                answer_b=ans2,
-                ref_answer_1=ans_ref,
-                **defaults)
+                question=ques, answer_a=ans1, answer_b=ans2, ref_answer_1=ans_ref, **defaults)
 
         return sys_prompt, user_prompt, output_format
 
     def get_review_cache(self, model_a, model_b, question) -> list:
         if model_b:
-            cache_hit = next(
-                (r for r in self.cache_list if r['model_a'] == model_a
-                 and r['model_b'] == model_b and r['question'] == question),
-                None)
+            cache_hit = next((r for r in self.cache_list
+                              if r['model_a'] == model_a and r['model_b'] == model_b and r['question'] == question),
+                             None)
         else:
-            cache_hit = next(
-                (r for r in self.cache_list
-                 if r['model'] == model_a and r['question'] == question), None)
+            cache_hit = next((r for r in self.cache_list if r['model'] == model_a and r['question'] == question), None)
         return cache_hit
 
     def get_review_pair(self, item: List[dict], dry_run=False, **kwargs) -> dict:
@@ -265,12 +241,10 @@ class AutoReviewerGpt4(BaseReviewer):
         return review_result
 
     def _get_review_pair(self, model_a, model_b, question, category, ans1, ans2, dry_run=False, **kwargs) -> (str, Any):
-        input_msg = dict(
-            ques=question, category=category, ans1=ans1, ans2=ans2)
+        input_msg = dict(ques=question, category=category, ans1=ans1, ans2=ans2)
 
         if self.reference_list:
-            ans_ref = next((ref for ref in self.reference_list
-                            if ref.get('text') == question), None)
+            ans_ref = next((ref for ref in self.reference_list if ref.get('text') == question), None)
             assert ans_ref['answer']
             input_msg['ans_ref'] = ans_ref['answer']
 
@@ -284,8 +258,7 @@ class AutoReviewerGpt4(BaseReviewer):
         else:
             review_text = self._get_reviewer_prediction(sys_prompt, user_prompt, **kwargs)
 
-        result = self.fn_completion_parser(
-            review_text, output_format=output_format)
+        result = self.fn_completion_parser(review_text, output_format=output_format)
         if not isinstance(result, tuple):
             result = (result, None)
         return review_text, *result
@@ -294,8 +267,7 @@ class AutoReviewerGpt4(BaseReviewer):
         input_msg = dict(ques=question, category=category, ans1=answer)
 
         if self.reference_list:
-            ans_ref = next((ref for ref in self.reference_list
-                            if ref.get('text') == question), None)
+            ans_ref = next((ref for ref in self.reference_list if ref.get('text') == question), None)
             assert ans_ref['answer']
             input_msg['ans_ref'] = ans_ref['answer']
 
@@ -312,8 +284,7 @@ class AutoReviewerGpt4(BaseReviewer):
         score = self.fn_completion_parser(review_text, output_format)
         return review_text, score
 
-    def _get_reviewer_prediction_dummy(self, sys_prompt: str, user_prompt: str,
-                                       output_format) -> str:
+    def _get_reviewer_prediction_dummy(self, sys_prompt: str, user_prompt: str, output_format) -> str:
         logger.info('Get dummy scores for input prompt ...')
         if output_format == '[[rating]]':
             return f'[[{round(random.random(), 2)}]]'
@@ -359,8 +330,7 @@ class AutoReviewerGpt4(BaseReviewer):
         if self.review_mode == ArenaMode.PAIRWISE:
             battle_pairs = get_battle_pairs(merged_ans_df.columns)
         elif self.review_mode == ArenaMode.PAIRWISE_BASELINE:
-            battle_pairs = get_battle_pairs(merged_ans_df.columns,
-                                            self.baseline_idx)
+            battle_pairs = get_battle_pairs(merged_ans_df.columns, self.baseline_idx)
         elif self.review_mode == ArenaMode.SINGLE:
             battle_pairs = [(col, ) for col in merged_ans_df.columns]
         else:
@@ -373,14 +343,12 @@ class AutoReviewerGpt4(BaseReviewer):
                 pair_df.columns = ['output_1', 'output_2']
                 pair_df['is_switched_outputs'] = pair_df.apply(
                     lambda x: random_seeded_choice(
-                        seed='is_switched_outputs' + x[0]['text'] + str(
-                            self.random_seed),
+                        seed='is_switched_outputs' + x[0]['text'] + str(self.random_seed),
                         choices=[False, True],
                     ),
                     axis=1,
                 )
-                pair_df = shuffle_pairwise_preferences(
-                    pair_df, pair_df['is_switched_outputs'])
+                pair_df = shuffle_pairwise_preferences(pair_df, pair_df['is_switched_outputs'])
 
             for index, row in pair_df.iterrows():
                 row_result = self.get_review_pair(row.to_list(), dry_run=dry_run, **kwargs) \
@@ -395,17 +363,21 @@ if __name__ == '__main__':
 
     work_path = os.path.join(Path(__file__).absolute().parent, '../../../')
     prompt_template_path = os.path.join(work_path, 'evalscope/registry/data/prompt_template/prompt_templates.jsonl')
-    answer_file_list = [os.path.join(work_path, 'outputs/arena/default/answers/answer_chatglm2-6b.jsonl'),
-                        os.path.join(work_path, 'outputs/arena/default/answers/answer_llama2-7b.jsonl')]
+    answer_file_list = [
+        os.path.join(work_path, 'outputs/arena/default/answers/answer_chatglm2-6b.jsonl'),
+        os.path.join(work_path, 'outputs/arena/default/answers/answer_llama2-7b.jsonl')
+    ]
     review_result_file_path = os.path.join(work_path, 'outputs/arena/default/reviews/review_gpt4.jsonl')
 
-    input_kwargs = dict(prompt_file=prompt_template_path,
-                        answer_file_list=answer_file_list,
-                        review_result_file=review_result_file_path,
-                        reviewer_args={},
-                        baseline_file='',
-                        reference_file='',
-                        cache_file='', )
+    input_kwargs = dict(
+        prompt_file=prompt_template_path,
+        answer_file_list=answer_file_list,
+        review_result_file=review_result_file_path,
+        reviewer_args={},
+        baseline_file='',
+        reference_file='',
+        cache_file='',
+    )
 
     auto_reviewer = AutoReviewerGpt4(**input_kwargs)
     auto_reviewer.run(dry_run=True)

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -29,16 +29,17 @@ In these examples settings.xml lists input files and formats.
 """
 
 from __future__ import absolute_import, division, print_function
-import collections
-import re
-import os
 
+import collections
 import nltk
 import numpy as np
+import os
+import re
 import six
 from absl import logging
 from rouge_score import scoring, tokenizers
 from six.moves import map, range
+
 from evalscope.utils import get_logger
 
 logger = get_logger()
@@ -81,11 +82,7 @@ class RougeScorer(scoring.BaseScorer):
         ...                       'The quick brown dog jumps on the log.')
     """
 
-    def __init__(self,
-                 rouge_types,
-                 use_stemmer=False,
-                 split_summaries=False,
-                 tokenizer=None):
+    def __init__(self, rouge_types, use_stemmer=False, split_summaries=False, tokenizer=None):
 
         self.rouge_types = rouge_types
         if tokenizer:
@@ -160,21 +157,15 @@ class RougeScorer(scoring.BaseScorer):
                     sents = [x for x in sents if len(x)]
                     return sents
 
-                target_tokens_list = [
-                    self._tokenizer.tokenize(s) for s in get_sents(target)
-                ]
-                prediction_tokens_list = [
-                    self._tokenizer.tokenize(s) for s in get_sents(prediction)
-                ]
+                target_tokens_list = [self._tokenizer.tokenize(s) for s in get_sents(target)]
+                prediction_tokens_list = [self._tokenizer.tokenize(s) for s in get_sents(prediction)]
 
-                scores = _summary_level_lcs(target_tokens_list,
-                                            prediction_tokens_list)
+                scores = _summary_level_lcs(target_tokens_list, prediction_tokens_list)
             elif re.match(r'rouge[0-9]$', six.ensure_str(rouge_type)):
                 # Rouge from n-grams.
                 n = int(rouge_type[5:])
                 if n <= 0:
-                    raise ValueError('rougen requires positive n: %s'
-                                     % rouge_type)
+                    raise ValueError('rougen requires positive n: %s' % rouge_type)
                 target_ngrams = _create_ngrams(target_tokens, n)
                 prediction_ngrams = _create_ngrams(prediction_tokens, n)
                 scores = _score_ngrams(target_ngrams, prediction_ngrams)
@@ -349,8 +340,7 @@ def _score_ngrams(target_ngrams, prediction_ngrams):
 
     intersection_ngrams_count = 0
     for ngram in six.iterkeys(target_ngrams):
-        intersection_ngrams_count += min(target_ngrams[ngram],
-                                         prediction_ngrams[ngram])
+        intersection_ngrams_count += min(target_ngrams[ngram], prediction_ngrams[ngram])
     target_ngrams_count = sum(target_ngrams.values())
     prediction_ngrams_count = sum(prediction_ngrams.values())
 

evalscope/metrics/code_metric.py

@@ -4,7 +4,6 @@ import inspect
 import re
 import signal
 from collections import defaultdict
-
 from tqdm import tqdm
 
 
@@ -20,8 +19,7 @@ def check_input(text, arg):
     code_block = code_block_pattern.search(text)
     code_string = code_block.group(1)
 
-    function_name_pattern = re.compile(r'def\s+([a-zA-Z_][a-zA-Z0-9_]*)\(',
-                                       re.DOTALL)
+    function_name_pattern = re.compile(r'def\s+([a-zA-Z_][a-zA-Z0-9_]*)\(', re.DOTALL)
     function_name_block = function_name_pattern.search(code_string)
     function_name = function_name_block.group(1)
 
@@ -52,9 +50,7 @@ def exec_func(func, arr):
 
 
 def compute_pass_k_one_sample(predict, func_args, func_outputs, k=4):
-    assert len(
-        predict
-    ) >= k, f'pass@k must have {k} generations, now have {len(predict)}'
+    assert len(predict) >= k, f'pass@k must have {k} generations, now have {len(predict)}'
     for predict_i in predict[:k]:
         try:
             for arg, gold in zip(func_args, func_outputs):
@@ -87,9 +83,7 @@ def compute_pass_k(predict_l, reference_l, func_args_l, k=4, lang='py'):
 def run_code_eval(data_l, k=4, md_level=2):
     print(f"{'#' * md_level} Code Eval(pass@{k})")
     for data in tqdm(data_l):
-        data[f'pass@{k}'] = compute_pass_k_one_sample(data['gen'],
-                                                      data['func_args'],
-                                                      data['func_outputs'], k)
+        data[f'pass@{k}'] = compute_pass_k_one_sample(data['gen'], data['func_args'], data['func_outputs'], k)
     task_data_d = defaultdict(list)
     for data in data_l:
         for task in data['task_tags']:

evalscope/metrics/math_accuracy.py

@@ -2,7 +2,6 @@
 
 import re
 from collections import defaultdict
-
 from tqdm import tqdm
 
 from evalscope.constants import MetricsConstant
@@ -44,8 +43,7 @@ def compute_math_accuracy(predict_l, reference_l):
 def run_math_eval(data_l, md_level=2):
     print(f"{'#' * md_level} Math Eval(math accuracy)")
     for data in tqdm(data_l):
-        data['math_accuracy'] = compute_math_accuracy_one_sample(
-            data['gen'], data['target'])
+        data['math_accuracy'] = compute_math_accuracy_one_sample(data['gen'], data['target'])
     task_data_d = defaultdict(list)
     for data in data_l:
         for task in data['task_tags']:
@@ -54,7 +52,6 @@ def run_math_eval(data_l, md_level=2):
     print(f'[total], count: {len(data_l)}, math accuracy: '
           f'{correct_cnt / len(data_l) * 100:0.2f}%')
     for task in task_data_d.keys():
-        correct_cnt = sum(
-            [data['math_accuracy'] for data in task_data_d[task]])
+        correct_cnt = sum([data['math_accuracy'] for data in task_data_d[task]])
         print(f'[{task}], count: {len(task_data_d[task])}, math accuracy: '
-              f'{correct_cnt/len(task_data_d[task])*100:0.2f}%')
+              f'{correct_cnt / len(task_data_d[task]) * 100:0.2f}%')

evalscope/metrics/metrics.py

@@ -2,18 +2,17 @@
 # Copyright (c) EleutherAI. and its affiliates.
 # Copyright (c) OpenAI. and its affiliates.
 import itertools
-import math
-from collections.abc import Iterable
-from collections import defaultdict
-from typing import Dict, List, Union
-from nltk.translate.bleu_score import sentence_bleu
-from nltk import word_tokenize
 import jieba
-
+import math
 import numpy as np
+import random
 import sacrebleu
 import sklearn.metrics
-import random
+from collections import defaultdict
+from collections.abc import Iterable
+from nltk import word_tokenize
+from nltk.translate.bleu_score import sentence_bleu
+from typing import Dict, List, Union
 
 
 def mean(arr):
@@ -22,12 +21,12 @@ def mean(arr):
 
 def pop_stddev(arr):
     mu = mean(arr)
-    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / len(arr))
+    return math.sqrt(sum([(x - mu)**2 for x in arr]) / len(arr))
 
 
 def sample_stddev(arr):
     mu = mean(arr)
-    return math.sqrt(sum([(x - mu) ** 2 for x in arr]) / (len(arr) - 1))
+    return math.sqrt(sum([(x - mu)**2 for x in arr]) / (len(arr) - 1))
 
 
 def mean_stderr(arr):
@@ -134,13 +133,14 @@ def bleu(items):
     refs, preds = _sacreformat(refs, preds)
     return sacrebleu.corpus_bleu(preds, refs).score
 
+
 def bleu_ngram_one_sample(predict, reference):
     """
     Calculate BLEU-1, BLEU-2, BLEU-3, and BLEU-4 scores
 
     Args:
         items: [(ref, pred)]
-        
+
     Returns:
         {
             'bleu-1': 0.8,
@@ -150,6 +150,7 @@ def bleu_ngram_one_sample(predict, reference):
         }
 
     """
+
     def is_contains_chinese(strs):
         for _char in strs:
             if '\u4e00' <= _char <= '\u9fa5':
@@ -230,6 +231,7 @@ def _sacreformat(refs, preds):
 
 
 class _bootstrap_internal:
+
     def __init__(self, f, n):
         self.f = f
         self.n = n
@@ -260,11 +262,11 @@ def bootstrap_stderr(f, xs, iters):
 
     print('bootstrapping for stddev:', f.__name__)
     for bootstrap in tqdm(
-        pool.imap(
-            _bootstrap_internal(f, chunk_size),
-            [(i, xs) for i in range(iters // chunk_size)],
-        ),
-        total=iters // chunk_size,
+            pool.imap(
+                _bootstrap_internal(f, chunk_size),
+                [(i, xs) for i in range(iters // chunk_size)],
+            ),
+            total=iters // chunk_size,
     ):
         # sample w replacement
         res.extend(bootstrap)
@@ -372,11 +374,9 @@ def calculate_arc_accuracy(question_answers: Dict[str, str], predictions: Dict[s
     return score / len(question_answers)
 
 
-def calculate_pass_at_k(
-    num_samples: Union[int, List[int], np.ndarray],
-    num_correct: Union[List[int], np.ndarray],
-    k: int = 1
-) -> np.ndarray:
+def calculate_pass_at_k(num_samples: Union[int, List[int], np.ndarray],
+                        num_correct: Union[List[int], np.ndarray],
+                        k: int = 1) -> np.ndarray:
     """
     Estimates pass@k of each problem and returns them in an array.
     Examples:

evalscope/metrics/rouge_metric.py

@@ -1,18 +1,16 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
+import jieba
 import logging
 from collections import defaultdict
 from pathlib import Path
+from rouge_chinese import Rouge
 from statistics import mean
-
 from tqdm import tqdm
 
 from evalscope.constants import MetricsConstant
 from evalscope.metrics.bundled_rouge_score import rouge_scorer
 
-from rouge_chinese import Rouge
-import jieba
-
 
 class DummyTokenizer:
 
@@ -24,9 +22,7 @@ HERE = Path(__file__).absolute().parent
 
 logger = logging.getLogger(__name__)
 
-scorer = rouge_scorer.RougeScorer(
-    ['rouge1', 'rouge2', 'rougeL'], tokenizer=DummyTokenizer()
-)
+scorer = rouge_scorer.RougeScorer(['rouge1', 'rouge2', 'rougeL'], tokenizer=DummyTokenizer())
 zh_scorer = Rouge()
 
 
@@ -52,11 +48,7 @@ def compute_rouge_score(predict_l, reference_l):
             result[rouge_key].append(one_sample[rouge_key])
     rlt = {}
     for rouge_key in MetricsConstant.ROUGE_KEYS:
-        rlt[rouge_key] = (
-            mean(result[rouge_key]) * 100
-            if rouge_key in result
-            else MetricsConstant.INVALID_VALUE
-        )
+        rlt[rouge_key] = (mean(result[rouge_key]) * 100 if rouge_key in result else MetricsConstant.INVALID_VALUE)
     return rlt
 
 
@@ -111,9 +103,9 @@ def _to_table(final_result) -> str:
             if not task:
                 continue
             elif task == 'total':
-                row.append(f'{final_result["total"]["rouge"][rouge_key] :0.2f}')
+                row.append(f'{final_result["total"]["rouge"][rouge_key]:0.2f}')
             else:
-                row.append(f'{final_result["tasks"][task]["rouge"][rouge_key] :0.2f}')
+                row.append(f'{final_result["tasks"][task]["rouge"][rouge_key]:0.2f}')
         table.append('\t'.join(row))
 
     return '\n'.join(table)
@@ -122,23 +114,17 @@ def _to_table(final_result) -> str:
 def run_rouge_eval(data_l, md_level=2, report_metric_key='rouge-l-f'):
     print(f"{'#' * md_level} Rouge Eval")
     for data in tqdm(data_l):
-        data['rouge'] = compute_rouge_score_one_sample(
-            data['gen_tok_str'], data['reference_tok_str']
-        )
+        data['rouge'] = compute_rouge_score_one_sample(data['gen_tok_str'], data['reference_tok_str'])
     task_data_d = defaultdict(list)
     for data in data_l:
         for task in data['task_tags']:
             task_data_d[task].append(data)
 
     total_rouge = mean([data['rouge'][report_metric_key] for data in data_l])
-    print(
-        f'[total], count: {len(data_l)}, {report_metric_key}: '
-        f'{total_rouge * 100:0.2f}%'
-    )
+    print(f'[total], count: {len(data_l)}, {report_metric_key}: '
+          f'{total_rouge * 100:0.2f}%')
 
     for task, task_data in task_data_d.items():
         task_rouge = mean([data['rouge'][report_metric_key] for data in task_data])
-        print(
-            f'[{task}], count: {len(task_data_d[task])}, {report_metric_key}: '
-            f'{task_rouge * 100:0.2f}%'
-        )
+        print(f'[{task}], count: {len(task_data_d[task])}, {report_metric_key}: '
+              f'{task_rouge * 100:0.2f}%')

evalscope/models/__init__.py

@@ -1,4 +1,3 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from evalscope.models.model import BaseModel
-from evalscope.models.model import ChatBaseModel
+from evalscope.models.model import BaseModel, ChatBaseModel