evalscope 0.5.0__py3-none-any.whl

evalscope/metrics/bundled_rouge_score/rouge_scorer.py

@@ -0,0 +1,342 @@
+# Copyright 2022 The rouge_score Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Computes rouge scores between two text blobs.
+
+Implementation replicates the functionality in the original ROUGE package. See:
+
+Lin, Chin-Yew. ROUGE: a Package for Automatic Evaluation of Summaries. In
+Proceedings of the Workshop on Text Summarization Branches Out (WAS 2004),
+Barcelona, Spain, July 25 - 26, 2004.
+
+Default options are equivalent to running:
+ROUGE-1.5.5.pl -e data -n 2 -a settings.xml
+
+Or with use_stemmer=True:
+ROUGE-1.5.5.pl -m -e data -n 2 -a settings.xml
+
+In these examples settings.xml lists input files and formats.
+"""
+
+from __future__ import absolute_import, division, print_function
+import collections
+import re
+
+import nltk
+import numpy as np
+import six
+from absl import logging
+from rouge_score import scoring, tokenizers
+from six.moves import map, range
+
+
+class RougeScorer(scoring.BaseScorer):
+    """
+    Calculate rouges scores between two blobs of text.
+
+    Args:
+        rouge_types: A list of rouge types to calculate.
+        use_stemmer: Bool indicating whether Porter stemmer should be used to
+            strip word suffixes to improve matching. This arg is used in the
+            DefaultTokenizer, but other tokenizers might or might not choose to
+            use this.
+        split_summaries: whether to add newlines between sentences for rougeLsum
+        tokenizer: Tokenizer object which has a tokenize() method.
+
+    Returns:
+      A dict mapping rouge types to Score tuples.
+
+    Examples:
+        >>> scorer = RougeScorer(['rouge1', 'rougeL'], use_stemmer=True)
+        >>> scores = scorer.score('The quick brown fox jumps over the lazy dog',
+        ...                       'The quick brown dog jumps on the log.')
+    """
+
+    def __init__(self,
+                 rouge_types,
+                 use_stemmer=False,
+                 split_summaries=False,
+                 tokenizer=None):
+
+        self.rouge_types = rouge_types
+        if tokenizer:
+            self._tokenizer = tokenizer
+        else:
+            self._tokenizer = tokenizers.DefaultTokenizer(use_stemmer)
+            logging.info('Using default tokenizer.')
+
+        self._split_summaries = split_summaries
+
+    def score_multi(self, targets, prediction):
+        """
+        Calculates rouge scores between targets and prediction.
+        The target with the maximum f-measure is used for the final score for each score type.
+
+        Args:
+            targets: list of texts containing the targets
+            prediction: Text containing the predicted text.
+
+        Returns:
+            A dict mapping each rouge type to a Score object.
+
+        Raises:
+            ValueError: If an invalid rouge type is encountered.
+        """
+
+        score_dicts = [self.score(t, prediction) for t in targets]
+        max_score = {}
+        for k in self.rouge_types:
+            index = np.argmax([s[k].fmeasure for s in score_dicts])
+            max_score[k] = score_dicts[index][k]
+
+        return max_score
+
+    def score(self, target, prediction):
+        """
+        Calculates rouge scores between the target and prediction.
+
+        Args:
+            target: Text containing the target (ground truth) text, or if a list
+            prediction: Text containing the predicted text.
+
+        Returns:
+            A dict mapping each rouge type to a Score object.
+
+        Raises:
+            ValueError: If an invalid rouge type is encountered.
+        """
+
+        # Pre-compute target tokens and prediction tokens for use by different
+        # types, except if only "rougeLsum" is requested.
+        if len(self.rouge_types) == 1 and self.rouge_types[0] == 'rougeLsum':
+            target_tokens = None
+            prediction_tokens = None
+        else:
+            target_tokens = self._tokenizer.tokenize(target)
+            prediction_tokens = self._tokenizer.tokenize(prediction)
+        result = {}
+
+        for rouge_type in self.rouge_types:
+            if rouge_type == 'rougeL':
+                # Rouge from longest common subsequences.
+                scores = _score_lcs(target_tokens, prediction_tokens)
+            elif rouge_type == 'rougeLsum':
+                # Note: Does not support multi-line text.
+                def get_sents(text):
+                    if self._split_summaries:
+                        sents = nltk.sent_tokenize(text)
+                    else:
+                        # Assume sentences are separated by newline.
+                        sents = six.ensure_str(text).split('\n')
+                    sents = [x for x in sents if len(x)]
+                    return sents
+
+                target_tokens_list = [
+                    self._tokenizer.tokenize(s) for s in get_sents(target)
+                ]
+                prediction_tokens_list = [
+                    self._tokenizer.tokenize(s) for s in get_sents(prediction)
+                ]
+
+                scores = _summary_level_lcs(target_tokens_list,
+                                            prediction_tokens_list)
+            elif re.match(r'rouge[0-9]$', six.ensure_str(rouge_type)):
+                # Rouge from n-grams.
+                n = int(rouge_type[5:])
+                if n <= 0:
+                    raise ValueError('rougen requires positive n: %s'
+                                     % rouge_type)
+                target_ngrams = _create_ngrams(target_tokens, n)
+                prediction_ngrams = _create_ngrams(prediction_tokens, n)
+                scores = _score_ngrams(target_ngrams, prediction_ngrams)
+            else:
+                raise ValueError('Invalid rouge type: %s' % rouge_type)
+            result[rouge_type] = scores
+
+        return result
+
+
+def _create_ngrams(tokens, n):
+    """
+    Creates ngrams from the given list of tokens.
+
+    Args:
+        tokens: A list of tokens from which ngrams are created.
+        n: Number of tokens to use, e.g. 2 for bigrams.
+
+    Returns:
+        A dictionary mapping each bigram to the number of occurrences.
+    """
+
+    ngrams = collections.Counter()
+    for ngram in (tuple(tokens[i:i + n]) for i in range(len(tokens) - n + 1)):
+        ngrams[ngram] += 1
+    return ngrams
+
+
+def _score_lcs(target_tokens, prediction_tokens):
+    """
+    Computes LCS (Longest Common Subsequence) rouge scores.
+
+    Args:
+        target_tokens: Tokens from the target text.
+        prediction_tokens: Tokens from the predicted text.
+
+    Returns:
+        A Score object containing computed scores.
+    """
+
+    if not target_tokens or not prediction_tokens:
+        return scoring.Score(precision=0, recall=0, fmeasure=0)
+
+    # Compute length of LCS from the bottom up in a table (DP appproach).
+    lcs_table = _lcs_table(target_tokens, prediction_tokens)
+    lcs_length = lcs_table[-1][-1]
+
+    precision = lcs_length / len(prediction_tokens)
+    recall = lcs_length / len(target_tokens)
+    fmeasure = scoring.fmeasure(precision, recall)
+
+    return scoring.Score(precision=precision, recall=recall, fmeasure=fmeasure)
+
+
+def _lcs_table(ref, can):
+    """Create 2-d LCS score table."""
+    rows = len(ref)
+    cols = len(can)
+    lcs_table = [[0] * (cols + 1) for _ in range(rows + 1)]
+    for i in range(1, rows + 1):
+        for j in range(1, cols + 1):
+            if ref[i - 1] == can[j - 1]:
+                lcs_table[i][j] = lcs_table[i - 1][j - 1] + 1
+            else:
+                lcs_table[i][j] = max(lcs_table[i - 1][j], lcs_table[i][j - 1])
+    return lcs_table
+
+
+def _backtrack_norec(t, ref, can):
+    """Read out LCS."""
+    i = len(ref)
+    j = len(can)
+    lcs = []
+    while i > 0 and j > 0:
+        if ref[i - 1] == can[j - 1]:
+            lcs.insert(0, i - 1)
+            i -= 1
+            j -= 1
+        elif t[i][j - 1] > t[i - 1][j]:
+            j -= 1
+        else:
+            i -= 1
+    return lcs
+
+
+def _summary_level_lcs(ref_sent, can_sent):
+    """
+    ROUGE: Summary-level LCS, section 3.2 in ROUGE paper.
+
+    Args:
+        ref_sent: list of tokenized reference sentences
+        can_sent: list of tokenized candidate sentences
+
+    Returns:
+        summary level ROUGE score
+    """
+
+    if not ref_sent or not can_sent:
+        return scoring.Score(precision=0, recall=0, fmeasure=0)
+
+    m = sum(map(len, ref_sent))
+    n = sum(map(len, can_sent))
+    if not n or not m:
+        return scoring.Score(precision=0, recall=0, fmeasure=0)
+
+    # get token counts to prevent double counting
+    token_cnts_r = collections.Counter()
+    token_cnts_c = collections.Counter()
+    for s in ref_sent:
+        # s is a list of tokens
+        token_cnts_r.update(s)
+    for s in can_sent:
+        token_cnts_c.update(s)
+
+    hits = 0
+    for r in ref_sent:
+        lcs = _union_lcs(r, can_sent)
+        # Prevent double-counting:
+        # The paper describes just computing hits += len(_union_lcs()),
+        # but the implementation prevents double counting. We also
+        # implement this as in version 1.5.5.
+        for t in lcs:
+            if token_cnts_c[t] > 0 and token_cnts_r[t] > 0:
+                hits += 1
+                token_cnts_c[t] -= 1
+                token_cnts_r[t] -= 1
+
+    recall = hits / m
+    precision = hits / n
+    fmeasure = scoring.fmeasure(precision, recall)
+    return scoring.Score(precision=precision, recall=recall, fmeasure=fmeasure)
+
+
+def _union_lcs(ref, c_list):
+    """
+    Find union LCS between a ref sentence and list of candidate sentences.
+
+    Args:
+        ref: list of tokens
+        c_list: list of list of indices for LCS into reference summary
+
+    Returns:
+        List of tokens in ref representing union LCS.
+    """
+
+    lcs_list = [lcs_ind(ref, c) for c in c_list]
+    return [ref[i] for i in _find_union(lcs_list)]
+
+
+def _find_union(lcs_list):
+    """Finds union LCS given a list of LCS."""
+    return sorted(list(set().union(*lcs_list)))
+
+
+def lcs_ind(ref, can):
+    """Returns one of the longest lcs."""
+    t = _lcs_table(ref, can)
+    return _backtrack_norec(t, ref, can)
+
+
+def _score_ngrams(target_ngrams, prediction_ngrams):
+    """
+    Computes n-gram based rouge scores.
+
+    Args:
+        target_ngrams: A Counter object mapping each ngram to number of occurrences for the target text.
+        prediction_ngrams: A Counter object mapping each ngram to number of occurrences for the prediction text.
+
+    Returns:
+        A Score object containing computed scores.
+    """
+
+    intersection_ngrams_count = 0
+    for ngram in six.iterkeys(target_ngrams):
+        intersection_ngrams_count += min(target_ngrams[ngram],
+                                         prediction_ngrams[ngram])
+    target_ngrams_count = sum(target_ngrams.values())
+    prediction_ngrams_count = sum(prediction_ngrams.values())
+
+    precision = intersection_ngrams_count / max(prediction_ngrams_count, 1)
+    recall = intersection_ngrams_count / max(target_ngrams_count, 1)
+    fmeasure = scoring.fmeasure(precision, recall)
+
+    return scoring.Score(precision=precision, recall=recall, fmeasure=fmeasure)

evalscope/metrics/code_metric.py

@@ -0,0 +1,104 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import inspect
+import re
+import signal
+from collections import defaultdict
+
+from tqdm import tqdm
+
+
+def handle(signum, frame):
+    raise RuntimeError('程序执行超过10秒')
+
+
+def check_input(text, arg):
+    pattern = r'input\((.*?)\n'
+    text = re.sub(pattern, '{}\n'.format(arg), text)
+
+    code_block_pattern = re.compile(r'```[Pp]ython\n(.*?)\n```', re.DOTALL)
+    code_block = code_block_pattern.search(text)
+    code_string = code_block.group(1)
+
+    function_name_pattern = re.compile(r'def\s+([a-zA-Z_][a-zA-Z0-9_]*)\(',
+                                       re.DOTALL)
+    function_name_block = function_name_pattern.search(code_string)
+    function_name = function_name_block.group(1)
+
+    return code_string, function_name
+
+
+def compile_func(code_string, function_name):
+    signal.signal(signal.SIGALRM, handle)
+    signal.alarm(10)
+    myMod = compile(code_string, '', 'exec')
+    exec(myMod)
+    func = eval(function_name)
+    signal.alarm(0)
+    return func
+
+
+def exec_func(func, arr):
+    signal.signal(signal.SIGALRM, handle)
+    signal.alarm(10)
+    sig = inspect.signature(func)
+    params = [param for param in sig.parameters]
+    if len(params) == 0:
+        result = func()
+    else:
+        result = func(arr)
+    signal.alarm(0)
+    return result
+
+
+def compute_pass_k_one_sample(predict, func_args, func_outputs, k=4):
+    assert len(
+        predict
+    ) >= k, f'pass@k must have {k} generations, now have {len(predict)}'
+    for predict_i in predict[:k]:
+        try:
+            for arg, gold in zip(func_args, func_outputs):
+                code_string, function_name = check_input(predict_i, arg)
+                func = compile_func(code_string, function_name)
+
+                exec_result = exec_func(func, arg)
+                if type(exec_result) is tuple:
+                    exec_result = list(exec_result)
+                exec_result = str(exec_result).lower()
+                assert exec_result == str(gold).lower()
+                del func
+            return 1
+        except Exception as e:
+            print(e)
+    return 0
+
+
+def compute_pass_k(predict_l, reference_l, func_args_l, k=4, lang='py'):
+    if lang != 'py':
+        print('only support python code.')
+
+    assert len(predict_l) == len(reference_l) == len(func_args_l)
+    pass_k_cnt = 0
+    for predict, ref, func_args in zip(predict_l, reference_l, func_args_l):
+        pass_k_cnt += compute_pass_k_one_sample(predict, ref, func_args, k)
+    return {'pass@k': pass_k_cnt / len(predict_l)}
+
+
+def run_code_eval(data_l, k=4, md_level=2):
+    print(f"{'#' * md_level} Code Eval(pass@{k})")
+    for data in tqdm(data_l):
+        data[f'pass@{k}'] = compute_pass_k_one_sample(data['gen'],
+                                                      data['func_args'],
+                                                      data['func_outputs'], k)
+    task_data_d = defaultdict(list)
+    for data in data_l:
+        for task in data['task_tags']:
+            task_data_d[task].append(data)
+
+    correct_cnt = sum([data[f'pass@{k}'] for data in data_l])
+    print(f'[total], count: {len(data_l)}, pass@{k}: '
+          f'{correct_cnt / len(data_l) * 100:0.2f}%')
+    for task in task_data_d.keys():
+        correct_cnt = sum([data[f'pass@{k}'] for data in task_data_d[task]])
+        print(f'[{task}], count: {len(task_data_d[task])}, pass@{k}: '
+              f'{correct_cnt / len(task_data_d[task]) * 100:0.2f}%')

evalscope/metrics/math_accuracy.py

@@ -0,0 +1,60 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+import re
+from collections import defaultdict
+
+from tqdm import tqdm
+
+from evalscope.constants import MetricsConstant
+
+
+def get_last_number(s):
+    match = re.search(r'[-+]?\d*\.\d+|\d+', s[::-1])
+    if match:
+        last_digit = match.group()[::-1]
+    else:
+        last_digit = -100000
+    return float(last_digit)
+
+
+def compute_math_accuracy_one_sample(predict, reference):
+    if isinstance(predict, list):
+        predict = predict[0]
+    if isinstance(reference, list):
+        reference = reference[0]
+    predict_number = get_last_number(predict)
+    reference_number = get_last_number(reference)
+    if abs(predict_number - reference_number) <= MetricsConstant.EPSILON:
+        return 1
+    else:
+        return 0
+
+
+def compute_math_accuracy(predict_l, reference_l):
+    assert len(predict_l) == len(reference_l)
+    if len(predict_l) == 0:
+        return 0
+    total_cnt = len(predict_l)
+    correct_cnt = 0
+    for predict, reference in zip(predict_l, reference_l):
+        correct_cnt += compute_math_accuracy_one_sample(predict, reference)
+    return {'math accuracy': correct_cnt / total_cnt}
+
+
+def run_math_eval(data_l, md_level=2):
+    print(f"{'#' * md_level} Math Eval(math accuracy)")
+    for data in tqdm(data_l):
+        data['math_accuracy'] = compute_math_accuracy_one_sample(
+            data['gen'], data['target'])
+    task_data_d = defaultdict(list)
+    for data in data_l:
+        for task in data['task_tags']:
+            task_data_d[task].append(data)
+    correct_cnt = sum([data['math_accuracy'] for data in data_l])
+    print(f'[total], count: {len(data_l)}, math accuracy: '
+          f'{correct_cnt / len(data_l) * 100:0.2f}%')
+    for task in task_data_d.keys():
+        correct_cnt = sum(
+            [data['math_accuracy'] for data in task_data_d[task]])
+        print(f'[{task}], count: {len(task_data_d[task])}, math accuracy: '
+              f'{correct_cnt/len(task_data_d[task])*100:0.2f}%')