evalscope 0.7.1__py3-none-any.whl → 0.8.0__py3-none-any.whl

evalscope/evaluator/humaneval_evaluator.py

@@ -0,0 +1,158 @@
+import json
+import os
+import re
+from tqdm import tqdm
+from typing import List, Optional
+
+from evalscope.constants import OutputsStructure
+from evalscope.evaluator.evaluator import logger
+from evalscope.models.model_adapter import BaseModelAdapter
+from evalscope.tools.combine_reports import gen_table
+from evalscope.utils import normalize_score
+
+
+class HumanevalEvaluator(object):
+
+    def __init__(
+        self,
+        problem_file: str,
+        model_id: str,
+        model_revision: str,
+        model_adapter: BaseModelAdapter,
+        outputs: Optional[OutputsStructure] = None,
+        k: List[int] = [1, 10, 100],
+        n_workers: int = 4,
+        timeout: float = 3.0,
+    ):
+        try:
+            from human_eval.data import read_problems, write_jsonl
+            from human_eval.evaluation import evaluate_functional_correctness
+        except ImportError:
+            raise ImportError('Please install human_eval:'
+                              'https://github.com/openai/human-eval/tree/master#installation , '
+                              'Note that you need to enable the execution code in the human_eval/execution.py first.')
+
+        self.problem_file = problem_file
+        self.k = k
+        self.num_workers = n_workers
+        self.timeout = timeout
+        self.model_adapter = model_adapter
+
+        self.read_problems_func = read_problems
+        self.write_jsonl_func = write_jsonl
+        self.eval_func = evaluate_functional_correctness
+
+        # {'task_id': {'task_id': '', 'prompt': '', 'entry_point': '', 'canonical_solution': '', 'test': ''}, ...}
+        self.problems = self.read_problems_func(self.problem_file)
+
+        # Deal with the output paths
+        self.outputs_structure = OutputsStructure(outputs)
+
+    def get_answers(self, infer_cfg: dict) -> List[dict]:
+        ans_list: list = []
+        system_prompt: str = 'Complete the following python code:\n'
+        for task_id, data_d in tqdm(self.problems.items(), total=len(self.problems), desc='Predicting(problems)'):
+            prompt: str = system_prompt + data_d['prompt']
+            inputs: dict = {'data': [prompt]}
+            # pred_res: dict = self.model_adapter.predict(inputs)
+
+            pred_res: dict = self.model_adapter.predict(inputs=inputs, infer_cfg=infer_cfg)
+
+            pred_ans: str = pred_res['choices'][0]['message']['content']
+            pred_ans = self._postprocess(pred_ans)
+
+            ans_list.append({'task_id': task_id, 'completion': pred_ans})
+
+        return ans_list
+
+    def eval(self, infer_cfg: dict, **kwargs):
+
+        # predict
+        ans_list: list = self.get_answers(infer_cfg)
+        ans_out_file: str = os.path.join(self.outputs_structure.predictions_dir, 'human_eval_predictions.jsonl')
+
+        self.write_jsonl_func(filename=ans_out_file, data=ans_list)
+        # logger.info(f'** Dump predictions to {ans_out_file} successfully.')
+        logger.info('** Dump predictions successfully.')
+
+        # evaluate  results: e.g. {'pass@1': 0.333, 'pass@10': 0.111}
+        results = self.eval_func(
+            sample_file=ans_out_file,
+            k=self.k,
+            n_workers=self.num_workers,
+            timeout=self.timeout,
+            problem_file=self.problem_file)
+
+        # output: report
+        report_map: dict = self.gen_report(results=results)
+        report_dir: str = self.outputs_structure.reports_dir
+        report_file: str = os.path.join(report_dir, 'human_eval_report.json')
+
+        with open(report_file, 'w') as f:
+            f.write(json.dumps(report_map, ensure_ascii=False, indent=4))
+        # logger.info(f'** Dump report to {report_file} \n')
+        logger.info('** Dump report \n')
+
+        try:
+            # Make table
+            report_table: str = gen_table([report_dir])
+            logger.info(f'** Report table: \n {report_table} \n')
+        except Exception:
+            logger.error('Failed to generate report table.')
+
+    def gen_report(self, results: dict) -> dict:
+        """
+        Generate report from evaluation results.
+
+        Returns:
+            {
+            "name":"ARC-Challenge",
+            "metric":"WeightedAverageAccuracy",
+            "score":0.3389,
+            "category":[
+                {
+                    "name":"DEFAULT",
+                    "score":0.3389,
+                    "subset":[
+                        {
+                            "name":"ARC-Challenge",
+                            "score":0.3389
+                        },
+                    ]
+                }
+            ],
+            "total_num":100
+        }
+        """
+        results = {k: normalize_score(score=v) for k, v in results.items()}
+
+        category_d = dict(name='DEFAULT', score=results, subset=[])
+
+        res_map = dict(
+            name='HumanEval', metric='pass@k', score=results, category=[category_d], total_num=len(self.problems))
+
+        return res_map
+
+    @classmethod
+    def _postprocess(cls, text: str) -> str:
+        if '```' in text:
+            blocks = re.findall(r'```(.*?)```', text, re.DOTALL)
+            if len(blocks) == 0:
+                text = text.split('```')[1]  # fall back to default strategy
+            else:
+                text = blocks[0]  # fetch the first code block
+                if not text.startswith('\n'):  # in case starting with ```python
+                    text = text[max(text.find('\n') + 1, 0):]
+        if text.strip().startswith('from') or text.strip().startswith('import'):
+            def_idx = text.find('def')
+            if def_idx != -1:
+                text = text[max(text.find('\n', def_idx) + 1, 0):]
+        text = text.split('\n\n')[0]
+        if text.strip().startswith('def'):
+            text = '\n'.join(text.split('\n')[1:])
+        if not text.startswith('    '):
+            if text.startswith(' '):
+                text = '    ' + text.lstrip()
+            else:
+                text = '\n'.join(['    ' + line for line in text.split('\n')])
+        return text

evalscope/evaluator/rating_eval.py

@@ -1,24 +1,17 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-from typing import List, Union
-
 import pandas as pd
 import pyarrow as pa
+from typing import List, Union
 
 from evalscope.constants import MetricMembers
+from evalscope.utils import jsonl_to_list
 from evalscope.utils.arena_utils import compute_elo
 from evalscope.utils.logger import get_logger
-from evalscope.utils import jsonl_to_list
 
 logger = get_logger()
 
-DEFAULT_COLUMNS_MAPPING = {
-    'model_a': 'model_a',
-    'model_b': 'model_b',
-    'win': 'win',
-    'tstamp': 'ts',
-    'language': 'lang'
-}
+DEFAULT_COLUMNS_MAPPING = {'model_a': 'model_a', 'model_b': 'model_b', 'win': 'win', 'tstamp': 'ts', 'language': 'lang'}
 
 
 class RatingEvaluate(object):
@@ -41,10 +34,9 @@ class RatingEvaluate(object):
         elo_ratings = compute_elo(battles)
         col_model = 'Model'
         col_elo_rating = 'Elo_Rating'
-        elo_ratings_res = pd.DataFrame(
-            [[n, elo_ratings[n]] for n in elo_ratings.keys()],
-            columns=[col_model, col_elo_rating]).sort_values(
-                col_elo_rating, ascending=False).reset_index(drop=True)
+        elo_ratings_res = pd.DataFrame([[n, elo_ratings[n]] for n in elo_ratings.keys()],
+                                       columns=[col_model, col_elo_rating]).sort_values(
+                                           col_elo_rating, ascending=False).reset_index(drop=True)
         elo_ratings_res = elo_ratings_res.round({col_elo_rating: 1})
         return elo_ratings_res
 
@@ -89,23 +81,11 @@ class RatingEvaluate(object):
                 'tie': 1
             }]
         else:
-            return [{
-                'model': winner,
-                'win': 1,
-                'loss': 0,
-                'tie': 0
-            }, {
-                'model': loser,
-                'win': 0,
-                'loss': 1,
-                'tie': 0
-            }]
+            return [{'model': winner, 'win': 1, 'loss': 0, 'tie': 0}, {'model': loser, 'win': 0, 'loss': 1, 'tie': 0}]
 
     def compute_pairwise_rating(self, raw_data):
         df_all = self.preprocess(raw_data_df=raw_data)
-        model_list = (
-            df_all['model_a'].unique().tolist()
-            + df_all['model_b'].unique().tolist())
+        model_list = (df_all['model_a'].unique().tolist() + df_all['model_b'].unique().tolist())
         model_list = list(set(model_list))
 
         list_res = []
@@ -114,8 +94,7 @@ class RatingEvaluate(object):
             if self.baseline_model is not None:
                 if self.baseline_model not in [row['model_a'], row['model_b']]:
                     logger.warning(
-                        f'One of the models in the battle should be the baseline model: {self.baseline_model}'
-                    )
+                        f'One of the models in the battle should be the baseline model: {self.baseline_model}')
                     continue
             rating = self.get_single_pairwise_rating(row)
             list_res = list_res + rating
@@ -149,15 +128,15 @@ class RatingEvaluate(object):
 
         for metric in self.metrics:
 
-            if metric == MetricMembers.ELO.value:
+            if metric == MetricMembers.ELO:
                 res = self.compute_elo_rating(raw_data)
                 res_all.append(res)
 
-            elif metric == MetricMembers.PAIRWISE.value:
+            elif metric == MetricMembers.PAIRWISE:
                 res = self.compute_pairwise_rating(raw_data)
                 res_all.append(res)
 
-            elif metric == MetricMembers.SCORE.value:
+            elif metric == MetricMembers.SCORE:
                 res = self.compute_score_rating(raw_data)
                 res_all.append(res)
 

evalscope/evaluator/reviewer/auto_reviewer.py

@@ -2,6 +2,7 @@
 # flake8: noqa
 
 import os
+import pandas as pd
 import random
 import sys
 import time
@@ -9,15 +10,10 @@ from abc import ABC, abstractmethod
 from functools import partial
 from typing import Any, List
 
-import pandas as pd
-
 from evalscope.constants import ArenaMode, EvalConfigKeys, FnCompletionParser, PositionBiasMitigation
 from evalscope.models.openai_model import OpenAIModel
-from evalscope.utils import completion_parsers
-from evalscope.utils.arena_utils import (get_battle_pairs,
-                                         merge_ques_ans,
-                                         shuffle_pairwise_preferences)
-from evalscope.utils import dump_jsonl_data, jsonl_to_list, random_seeded_choice
+from evalscope.utils import completion_parsers, dump_jsonl_data, jsonl_to_list, random_seeded_choice
+from evalscope.utils.arena_utils import get_battle_pairs, merge_ques_ans, shuffle_pairwise_preferences
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -33,8 +29,7 @@ class BaseReviewer(ABC):
         """
         Run pairwise battles with given models.
         """
-        raise NotImplementedError(
-            'run() method must be implemented in your subclass.')
+        raise NotImplementedError('run() method must be implemented in your subclass.')
 
 
 class AutoReviewerGpt4(BaseReviewer):
@@ -71,13 +66,9 @@ class AutoReviewerGpt4(BaseReviewer):
 
         self.review_result_file = review_result_file
         self.prompt_list = jsonl_to_list(prompt_file)
-        self.answer_list = [
-            jsonl_to_list(answer_file) for answer_file in answer_file_list
-        ]
-        self.reference_list = jsonl_to_list(
-            reference_file) if reference_file else []
-        self.cache_list = jsonl_to_list(
-            cache_file) if cache_file and os.path.isfile(cache_file) else []
+        self.answer_list = [jsonl_to_list(answer_file) for answer_file in answer_file_list]
+        self.reference_list = jsonl_to_list(reference_file) if reference_file else []
+        self.cache_list = jsonl_to_list(cache_file) if cache_file and os.path.isfile(cache_file) else []
 
         self.reviewer_args = reviewer_args if reviewer_args \
             else self._get_default_args()
@@ -88,24 +79,18 @@ class AutoReviewerGpt4(BaseReviewer):
             self.answer_list.append(jsonl_to_list(baseline_file))
             self.baseline_idx = len(self.answer_list) - 1
 
-        self.position_bias_mitigation = self.reviewer_args.pop(
-            EvalConfigKeys.POSITION_BIAS_MITIGATION,
-            PositionBiasMitigation.NONE)
+        self.position_bias_mitigation = self.reviewer_args.pop(EvalConfigKeys.POSITION_BIAS_MITIGATION,
+                                                               PositionBiasMitigation.NONE)
         if self.position_bias_mitigation == PositionBiasMitigation.RANDOMIZE_ORDER:
-            self.random_seed = self.reviewer_args.pop(
-                EvalConfigKeys.RANDOM_SEED, 123)
-
-        fn_completion_parser = self.reviewer_args.pop(
-            EvalConfigKeys.FN_COMPLETION_PARSER,
-            FnCompletionParser.LMSYS_PARSER)
-        completion_parser_kwargs = self.reviewer_args.pop(
-            EvalConfigKeys.COMPLETION_PARSER_KWARGS, {})
+            self.random_seed = self.reviewer_args.pop(EvalConfigKeys.RANDOM_SEED, 123)
+
+        fn_completion_parser = self.reviewer_args.pop(EvalConfigKeys.FN_COMPLETION_PARSER,
+                                                      FnCompletionParser.LMSYS_PARSER)
+        completion_parser_kwargs = self.reviewer_args.pop(EvalConfigKeys.COMPLETION_PARSER_KWARGS, {})
         if isinstance(fn_completion_parser, str):
-            fn_completion_parser = getattr(completion_parsers,
-                                           fn_completion_parser)
+            fn_completion_parser = getattr(completion_parsers, fn_completion_parser)
 
-        self.fn_completion_parser = partial(fn_completion_parser,
-                                            **completion_parser_kwargs)
+        self.fn_completion_parser = partial(fn_completion_parser, **completion_parser_kwargs)
         self.gpt_predictor = OpenAIModel(model_cfg=self.reviewer_args)
 
     @staticmethod
@@ -133,45 +118,35 @@ class AutoReviewerGpt4(BaseReviewer):
         # Default to general category (idx 0)
         target_prompt_dict = prompts_list[0]
         for item in prompts_list:
-            is_category_match = category in item['category'] if isinstance(
-                item['category'], list) else item['category'] == category
+            is_category_match = category in item['category'] if isinstance(item['category'],
+                                                                           list) else item['category'] == category
             is_type_match = item.get('type', ArenaMode.PAIRWISE) == type
             if is_category_match and is_type_match:
                 target_prompt_dict = item
                 break
-            elif is_type_match and target_prompt_dict.get('type',
-                                                          ArenaMode.PAIRWISE) != type:
+            elif is_type_match and target_prompt_dict.get('type', ArenaMode.PAIRWISE) != type:
                 target_prompt_dict = item  # fallback to type match
 
         sys_prompt = target_prompt_dict['system_prompt']
         prompt_template = target_prompt_dict['prompt_template']
         defaults = target_prompt_dict.get('defaults', dict({}))
-        output_format = target_prompt_dict.get('output_format',
-                                               '[[rating_a,rating_b]]')
+        output_format = target_prompt_dict.get('output_format', '[[rating_a,rating_b]]')
 
         if type == ArenaMode.SINGLE:
-            user_prompt = prompt_template.format(
-                question=ques, answer=ans1, ref_answer_1=ans_ref, **defaults)
+            user_prompt = prompt_template.format(question=ques, answer=ans1, ref_answer_1=ans_ref, **defaults)
         else:
             user_prompt = prompt_template.format(
-                question=ques,
-                answer_a=ans1,
-                answer_b=ans2,
-                ref_answer_1=ans_ref,
-                **defaults)
+                question=ques, answer_a=ans1, answer_b=ans2, ref_answer_1=ans_ref, **defaults)
 
         return sys_prompt, user_prompt, output_format
 
     def get_review_cache(self, model_a, model_b, question) -> list:
         if model_b:
-            cache_hit = next(
-                (r for r in self.cache_list if r['model_a'] == model_a
-                 and r['model_b'] == model_b and r['question'] == question),
-                None)
+            cache_hit = next((r for r in self.cache_list
+                              if r['model_a'] == model_a and r['model_b'] == model_b and r['question'] == question),
+                             None)
         else:
-            cache_hit = next(
-                (r for r in self.cache_list
-                 if r['model'] == model_a and r['question'] == question), None)
+            cache_hit = next((r for r in self.cache_list if r['model'] == model_a and r['question'] == question), None)
         return cache_hit
 
     def get_review_pair(self, item: List[dict], dry_run=False, **kwargs) -> dict:
@@ -265,12 +240,10 @@ class AutoReviewerGpt4(BaseReviewer):
         return review_result
 
     def _get_review_pair(self, model_a, model_b, question, category, ans1, ans2, dry_run=False, **kwargs) -> (str, Any):
-        input_msg = dict(
-            ques=question, category=category, ans1=ans1, ans2=ans2)
+        input_msg = dict(ques=question, category=category, ans1=ans1, ans2=ans2)
 
         if self.reference_list:
-            ans_ref = next((ref for ref in self.reference_list
-                            if ref.get('text') == question), None)
+            ans_ref = next((ref for ref in self.reference_list if ref.get('text') == question), None)
             assert ans_ref['answer']
             input_msg['ans_ref'] = ans_ref['answer']
 
@@ -284,8 +257,7 @@ class AutoReviewerGpt4(BaseReviewer):
         else:
             review_text = self._get_reviewer_prediction(sys_prompt, user_prompt, **kwargs)
 
-        result = self.fn_completion_parser(
-            review_text, output_format=output_format)
+        result = self.fn_completion_parser(review_text, output_format=output_format)
         if not isinstance(result, tuple):
             result = (result, None)
         return review_text, *result
@@ -294,8 +266,7 @@ class AutoReviewerGpt4(BaseReviewer):
         input_msg = dict(ques=question, category=category, ans1=answer)
 
         if self.reference_list:
-            ans_ref = next((ref for ref in self.reference_list
-                            if ref.get('text') == question), None)
+            ans_ref = next((ref for ref in self.reference_list if ref.get('text') == question), None)
             assert ans_ref['answer']
             input_msg['ans_ref'] = ans_ref['answer']
 
@@ -312,8 +283,7 @@ class AutoReviewerGpt4(BaseReviewer):
         score = self.fn_completion_parser(review_text, output_format)
         return review_text, score
 
-    def _get_reviewer_prediction_dummy(self, sys_prompt: str, user_prompt: str,
-                                       output_format) -> str:
+    def _get_reviewer_prediction_dummy(self, sys_prompt: str, user_prompt: str, output_format) -> str:
         logger.info('Get dummy scores for input prompt ...')
         if output_format == '[[rating]]':
             return f'[[{round(random.random(), 2)}]]'
@@ -359,8 +329,7 @@ class AutoReviewerGpt4(BaseReviewer):
         if self.review_mode == ArenaMode.PAIRWISE:
             battle_pairs = get_battle_pairs(merged_ans_df.columns)
         elif self.review_mode == ArenaMode.PAIRWISE_BASELINE:
-            battle_pairs = get_battle_pairs(merged_ans_df.columns,
-                                            self.baseline_idx)
+            battle_pairs = get_battle_pairs(merged_ans_df.columns, self.baseline_idx)
         elif self.review_mode == ArenaMode.SINGLE:
             battle_pairs = [(col, ) for col in merged_ans_df.columns]
         else:
@@ -373,14 +342,12 @@ class AutoReviewerGpt4(BaseReviewer):
                 pair_df.columns = ['output_1', 'output_2']
                 pair_df['is_switched_outputs'] = pair_df.apply(
                     lambda x: random_seeded_choice(
-                        seed='is_switched_outputs' + x[0]['text'] + str(
-                            self.random_seed),
+                        seed='is_switched_outputs' + x[0]['text'] + str(self.random_seed),
                         choices=[False, True],
                     ),
                     axis=1,
                 )
-                pair_df = shuffle_pairwise_preferences(
-                    pair_df, pair_df['is_switched_outputs'])
+                pair_df = shuffle_pairwise_preferences(pair_df, pair_df['is_switched_outputs'])
 
             for index, row in pair_df.iterrows():
                 row_result = self.get_review_pair(row.to_list(), dry_run=dry_run, **kwargs) \
@@ -395,17 +362,21 @@ if __name__ == '__main__':
 
     work_path = os.path.join(Path(__file__).absolute().parent, '../../../')
     prompt_template_path = os.path.join(work_path, 'evalscope/registry/data/prompt_template/prompt_templates.jsonl')
-    answer_file_list = [os.path.join(work_path, 'outputs/arena/default/answers/answer_chatglm2-6b.jsonl'),
-                        os.path.join(work_path, 'outputs/arena/default/answers/answer_llama2-7b.jsonl')]
+    answer_file_list = [
+        os.path.join(work_path, 'outputs/arena/default/answers/answer_chatglm2-6b.jsonl'),
+        os.path.join(work_path, 'outputs/arena/default/answers/answer_llama2-7b.jsonl')
+    ]
     review_result_file_path = os.path.join(work_path, 'outputs/arena/default/reviews/review_gpt4.jsonl')
 
-    input_kwargs = dict(prompt_file=prompt_template_path,
-                        answer_file_list=answer_file_list,
-                        review_result_file=review_result_file_path,
-                        reviewer_args={},
-                        baseline_file='',
-                        reference_file='',
-                        cache_file='', )
+    input_kwargs = dict(
+        prompt_file=prompt_template_path,
+        answer_file_list=answer_file_list,
+        review_result_file=review_result_file_path,
+        reviewer_args={},
+        baseline_file='',
+        reference_file='',
+        cache_file='',
+    )
 
     auto_reviewer = AutoReviewerGpt4(**input_kwargs)
     auto_reviewer.run(dry_run=True)

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -29,16 +29,17 @@ In these examples settings.xml lists input files and formats.
 """
 
 from __future__ import absolute_import, division, print_function
-import collections
-import re
-import os
 
+import collections
 import nltk
 import numpy as np
+import os
+import re
 import six
 from absl import logging
 from rouge_score import scoring, tokenizers
 from six.moves import map, range
+
 from evalscope.utils import get_logger
 
 logger = get_logger()
@@ -81,11 +82,7 @@ class RougeScorer(scoring.BaseScorer):
         ...                       'The quick brown dog jumps on the log.')
     """
 
-    def __init__(self,
-                 rouge_types,
-                 use_stemmer=False,
-                 split_summaries=False,
-                 tokenizer=None):
+    def __init__(self, rouge_types, use_stemmer=False, split_summaries=False, tokenizer=None):
 
         self.rouge_types = rouge_types
         if tokenizer:
@@ -160,21 +157,15 @@ class RougeScorer(scoring.BaseScorer):
                     sents = [x for x in sents if len(x)]
                     return sents
 
-                target_tokens_list = [
-                    self._tokenizer.tokenize(s) for s in get_sents(target)
-                ]
-                prediction_tokens_list = [
-                    self._tokenizer.tokenize(s) for s in get_sents(prediction)
-                ]
+                target_tokens_list = [self._tokenizer.tokenize(s) for s in get_sents(target)]
+                prediction_tokens_list = [self._tokenizer.tokenize(s) for s in get_sents(prediction)]
 
-                scores = _summary_level_lcs(target_tokens_list,
-                                            prediction_tokens_list)
+                scores = _summary_level_lcs(target_tokens_list, prediction_tokens_list)
             elif re.match(r'rouge[0-9]$', six.ensure_str(rouge_type)):
                 # Rouge from n-grams.
                 n = int(rouge_type[5:])
                 if n <= 0:
-                    raise ValueError('rougen requires positive n: %s'
-                                     % rouge_type)
+                    raise ValueError('rougen requires positive n: %s' % rouge_type)
                 target_ngrams = _create_ngrams(target_tokens, n)
                 prediction_ngrams = _create_ngrams(prediction_tokens, n)
                 scores = _score_ngrams(target_ngrams, prediction_ngrams)
@@ -349,8 +340,7 @@ def _score_ngrams(target_ngrams, prediction_ngrams):
 
     intersection_ngrams_count = 0
     for ngram in six.iterkeys(target_ngrams):
-        intersection_ngrams_count += min(target_ngrams[ngram],
-                                         prediction_ngrams[ngram])
+        intersection_ngrams_count += min(target_ngrams[ngram], prediction_ngrams[ngram])
     target_ngrams_count = sum(target_ngrams.values())
     prediction_ngrams_count = sum(prediction_ngrams.values())
 

evalscope/metrics/code_metric.py

@@ -4,7 +4,6 @@ import inspect
 import re
 import signal
 from collections import defaultdict
-
 from tqdm import tqdm
 
 
@@ -20,8 +19,7 @@ def check_input(text, arg):
     code_block = code_block_pattern.search(text)
     code_string = code_block.group(1)
 
-    function_name_pattern = re.compile(r'def\s+([a-zA-Z_][a-zA-Z0-9_]*)\(',
-                                       re.DOTALL)
+    function_name_pattern = re.compile(r'def\s+([a-zA-Z_][a-zA-Z0-9_]*)\(', re.DOTALL)
     function_name_block = function_name_pattern.search(code_string)
     function_name = function_name_block.group(1)
 
@@ -52,9 +50,7 @@ def exec_func(func, arr):
 
 
 def compute_pass_k_one_sample(predict, func_args, func_outputs, k=4):
-    assert len(
-        predict
-    ) >= k, f'pass@k must have {k} generations, now have {len(predict)}'
+    assert len(predict) >= k, f'pass@k must have {k} generations, now have {len(predict)}'
     for predict_i in predict[:k]:
         try:
             for arg, gold in zip(func_args, func_outputs):
@@ -87,9 +83,7 @@ def compute_pass_k(predict_l, reference_l, func_args_l, k=4, lang='py'):
 def run_code_eval(data_l, k=4, md_level=2):
     print(f"{'#' * md_level} Code Eval(pass@{k})")
     for data in tqdm(data_l):
-        data[f'pass@{k}'] = compute_pass_k_one_sample(data['gen'],
-                                                      data['func_args'],
-                                                      data['func_outputs'], k)
+        data[f'pass@{k}'] = compute_pass_k_one_sample(data['gen'], data['func_args'], data['func_outputs'], k)
     task_data_d = defaultdict(list)
     for data in data_l:
         for task in data['task_tags']:

evalscope/metrics/math_accuracy.py

@@ -2,7 +2,6 @@
 
 import re
 from collections import defaultdict
-
 from tqdm import tqdm
 
 from evalscope.constants import MetricsConstant
@@ -44,8 +43,7 @@ def compute_math_accuracy(predict_l, reference_l):
 def run_math_eval(data_l, md_level=2):
     print(f"{'#' * md_level} Math Eval(math accuracy)")
     for data in tqdm(data_l):
-        data['math_accuracy'] = compute_math_accuracy_one_sample(
-            data['gen'], data['target'])
+        data['math_accuracy'] = compute_math_accuracy_one_sample(data['gen'], data['target'])
     task_data_d = defaultdict(list)
     for data in data_l:
         for task in data['task_tags']:
@@ -54,7 +52,6 @@ def run_math_eval(data_l, md_level=2):
     print(f'[total], count: {len(data_l)}, math accuracy: '
           f'{correct_cnt / len(data_l) * 100:0.2f}%')
     for task in task_data_d.keys():
-        correct_cnt = sum(
-            [data['math_accuracy'] for data in task_data_d[task]])
+        correct_cnt = sum([data['math_accuracy'] for data in task_data_d[task]])
         print(f'[{task}], count: {len(task_data_d[task])}, math accuracy: '
-              f'{correct_cnt/len(task_data_d[task])*100:0.2f}%')
+              f'{correct_cnt / len(task_data_d[task]) * 100:0.2f}%')