evalscope 1.0.2__py3-none-any.whl → 1.1.1__py3-none-any.whl

evalscope/benchmarks/omnidoc_bench/metrics.py

@@ -0,0 +1,547 @@
+# flake8: noqa
+import copy
+import jieba
+import Levenshtein
+import pandas as pd
+import re
+from apted import APTED, Config
+from apted.helpers import Tree
+from collections import defaultdict, deque
+from lxml import etree, html
+from nltk.translate.bleu_score import corpus_bleu
+from nltk.translate.meteor_score import meteor_score
+from tabulate import tabulate
+from tqdm import tqdm
+
+from evalscope.utils import get_logger
+from .utils import normalized_table
+
+logger = get_logger()
+
+
+def show_result(results):
+    for metric_name in results.keys():
+        score_table = [[k, v] for k, v in results[metric_name].items()]
+        logger.info(f'\n{metric_name}:\n' + tabulate(score_table) + '\n' + '=' * 100)
+
+
+def sort_nested_dict(d):
+    # If it's a dictionary, recursively sort it
+    if isinstance(d, dict):
+        # Sort the current dictionary
+        sorted_dict = {k: sort_nested_dict(v) for k, v in sorted(d.items())}
+        return sorted_dict
+    # If not a dictionary, return directly
+    return d
+
+
+def get_full_labels_results(samples: dict):
+    if not samples:
+        return {}
+    label_group_dict = defaultdict(lambda: defaultdict(list))
+    for sample in samples:
+        label_list = []
+        if not sample.get('gt_attribute'):
+            continue
+        for anno in sample['gt_attribute']:
+            for k, v in anno.items():
+                label_list.append(k + ': ' + str(v))
+        for label_name in list(
+            set(label_list)
+        ):  # Currently if there are merged cases, calculate based on the set of all labels involved after merging
+            for metric, score in sample['metric'].items():
+                label_group_dict[label_name][metric].append(score)
+
+    logger.info('----Anno Attribute---------------')
+    result = {}
+    result['sample_count'] = {}
+    for attribute in label_group_dict.keys():
+        for metric, scores in label_group_dict[attribute].items():
+            mean_score = sum(scores) / len(scores)
+            if not result.get(metric):
+                result[metric] = {}
+            result[metric][attribute] = mean_score
+            result['sample_count'][attribute] = len(scores)
+    result = sort_nested_dict(result)
+    show_result(result)
+    return result
+
+
+def get_page_split(samples, page_info):  # Page level metric
+    if not page_info:
+        return {}
+    result_list = defaultdict(list)
+
+    for sample in samples:
+        img_name = sample['img_id'] if sample['img_id'].endswith('.jpg') else '_'.join(sample['img_id'].split('_')[:-1])
+        page_info_s = page_info[img_name]
+        if not sample.get('metric'):
+            continue
+        for metric, score in sample['metric'].items():
+            gt = sample['norm_gt'] if sample.get('norm_gt') else sample['gt']
+            pred = sample['norm_pred'] if sample.get('norm_pred') else sample['pred']
+            result_list[metric].append({
+                'image_name': img_name,
+                'metric': metric,
+                'attribute': 'ALL',
+                'score': score,
+                'upper_len': max(len(gt), len(pred))
+            })
+            for k, v in page_info_s.items():
+                if isinstance(v, list):  # special issue
+                    for special_issue in v:
+                        if 'table' not in special_issue:  # Table-related special fields have duplicates
+                            result_list[metric].append({
+                                'image_name': img_name,
+                                'metric': metric,
+                                'attribute': special_issue,
+                                'score': score,
+                                'upper_len': max(len(gt), len(pred))
+                            })
+                else:
+                    result_list[metric].append({
+                        'image_name': img_name,
+                        'metric': metric,
+                        'attribute': k + ': ' + str(v),
+                        'score': score,
+                        'upper_len': max(len(gt), len(pred))
+                    })
+
+    # Page level logic, accumulation is only done within pages, and mean operation is performed between pages
+    result = {}
+    if result_list.get('Edit_dist'):
+        df = pd.DataFrame(result_list['Edit_dist'])
+        up_total_avg = df.groupby(
+            ['image_name', 'attribute']
+        ).apply(lambda x: (x['score'] * x['upper_len']).sum() / x['upper_len'].sum(),
+                include_groups=False).groupby('attribute').mean(
+                )  # At page level, accumulate edits, denominator is sum of max(gt, pred) from each sample
+        result['Edit_dist'] = up_total_avg.to_dict()
+    for metric in result_list.keys():
+        if metric == 'Edit_dist':
+            continue
+        df = pd.DataFrame(result_list[metric])
+        page_avg = df.groupby(['image_name', 'attribute']).apply(lambda x: x['score'].mean(),
+                                                                 include_groups=False).groupby('attribute').mean()
+        result[metric] = page_avg.to_dict()
+
+    result = sort_nested_dict(result)
+    # print('----Page Attribute---------------')
+    show_result(result)
+    return result
+
+
+def get_groups(samples, group_info):
+    group_samples = defaultdict(list)
+    for sample in samples:
+        group_samples['all'].append(sample)
+        for group in group_info:
+            select_flag = True
+            for k, v in group.items():
+                for gt_attribute in sample['gt_attribute'
+                                           ]:  # gt_attribute is a list containing all merged gt attributes
+                    if not gt_attribute:  # if no GT attributes, don't include in calculation
+                        select_flag = False
+                    elif gt_attribute[k] != v:  # if any gt attribute doesn't meet criteria, don't select
+                        select_flag = False
+            if select_flag:
+                group_samples[str(group)].append(sample)
+    return group_samples
+
+
+class Registry:
+
+    def __init__(self):
+        self._registry = {}
+
+    def register(self, name):
+
+        def decorator(item):
+            if name in self._registry:
+                raise ValueError(f'Item {name} already registered.')
+            self._registry[name] = item
+            return item
+
+        return decorator
+
+    def get(self, name):
+        if name not in self._registry:
+            raise ValueError(f'Item {name} not found in registry.')
+        return self._registry[name]
+
+    def list_items(self):
+        return list(self._registry.keys())
+
+
+METRIC_REGISTRY = Registry()
+
+
+@METRIC_REGISTRY.register('TEDS')
+class call_TEDS():
+
+    def __init__(self, samples):
+        self.samples = samples
+
+    def evaluate(self, group_info=[], save_name='default'):
+        teds = TEDS(structure_only=False)
+        teds_structure_only = TEDS(structure_only=True)
+
+        group_scores = defaultdict(list)
+        group_scores_structure_only = defaultdict(list)
+
+        samples = self.samples
+        for sample in samples:
+            gt = sample['norm_gt'] if sample.get('norm_gt') else sample['gt']
+            pred = sample['norm_pred'] if sample.get('norm_pred') else sample['pred']
+
+            score = teds.evaluate(pred, gt)
+            score_structure_only = teds_structure_only.evaluate(pred, gt)
+            # print('TEDS score:', score)
+            group_scores['all'].append(score)
+            group_scores_structure_only['all'].append(score_structure_only)
+
+            if not sample.get('metric'):
+                sample['metric'] = {}
+            sample['metric']['TEDS'] = score
+            sample['metric']['TEDS_structure_only'] = score_structure_only
+
+            for group in group_info:
+                select_flag = True
+                for k, v in group.items():
+                    for gt_attribute in sample['gt_attribute'
+                                               ]:  # gt_attribute is a list containing all merged gt attributes
+                        if not gt_attribute:  # if no GT attributes, don't include in calculation
+                            select_flag = False
+                        elif gt_attribute[k] != v:  # if any gt attribute doesn't meet criteria, don't select
+                            select_flag = False
+                if select_flag:
+                    group_scores[str(group)].append(score)
+
+        result = {}
+        for group_name, scores in group_scores.items():
+            if len(scores) > 0:
+                result[group_name] = sum(scores) / len(scores)  # average of normalized scores at sample level
+            else:
+                result[group_name] = 'NaN'
+                logger.warning(f'Empty matched samples for {group_name}.')
+
+        structure_only_result = {}
+        for group_name, scores in group_scores_structure_only.items():
+            if len(scores) > 0:
+                structure_only_result[group_name] = sum(scores) / len(
+                    scores
+                )  # average of normalized scores at sample level
+            else:
+                structure_only_result[group_name] = 'NaN'
+                logger.warning(f'Empty matched samples for {group_name}.')
+
+        return samples, {'TEDS': result, 'TEDS_structure_only': structure_only_result}
+
+
+def tokenize(text) -> list[str]:
+    """Tokenizes text, handling Chinese and non-Chinese strings appropriately."""
+
+    def contain_chinese_string(text):
+        chinese_pattern = re.compile(r'[\u4e00-\u9fa5]')
+        return bool(chinese_pattern.search(text))
+
+    if contain_chinese_string(text):
+        res = jieba.lcut(text)
+    else:
+        res = text.split()
+    return res
+
+
+@METRIC_REGISTRY.register('BLEU')
+class call_BLEU():
+
+    def __init__(self, samples):
+        self.samples = samples
+
+    def evaluate(self, group_info=[], save_name='default'):
+        group_samples = get_groups(self.samples, group_info)
+        result = {}
+
+        for group_name, samples in group_samples.items():
+            predictions, references = [], []
+            for sample in samples:
+                gt = sample['norm_gt'] if sample.get('norm_gt') else sample['gt']
+                pred = sample['norm_pred'] if sample.get('norm_pred') else sample['pred']
+                predictions.append(tokenize(pred))
+                references.append([tokenize(gt)])
+
+            if not predictions or not any(predictions) or not references or not any(references):
+                bleu_score = 0
+            else:
+                try:
+                    bleu_score = corpus_bleu(references, predictions)
+                except ZeroDivisionError:
+                    bleu_score = 0
+
+            result[group_name] = bleu_score
+
+        return self.samples, {'BLEU': result}
+
+
+@METRIC_REGISTRY.register('METEOR')
+class call_METEOR():
+
+    def __init__(self, samples):
+        self.samples = samples
+
+    def evaluate(self, group_info=[], save_name='default'):
+        group_samples = get_groups(self.samples, group_info)
+        result = {}
+        for group_name, samples in group_samples.items():
+            predictions, references = [], []
+            for sample in samples:
+                gt = sample['norm_gt'] if sample.get('norm_gt') else sample['gt']
+                pred = sample['norm_pred'] if sample.get('norm_pred') else sample['pred']
+                predictions.append(tokenize(pred))
+                references.append(tokenize(gt))
+            # Calculate METEOR score
+            if not predictions or not references:
+                meteor_results = 0
+            else:
+                try:
+                    total_score = 0
+                    for ref, pred in zip(references, predictions):
+                        score = meteor_score([ref], pred)
+                        total_score += score
+                    meteor_results = total_score / len(references)
+                except Exception as e:
+                    logger.error(f'METEOR calculation error: {e}')
+                    meteor_results = 0
+            result[group_name] = meteor_results
+
+        return self.samples, {'METEOR': result}
+
+
+@METRIC_REGISTRY.register('Edit_dist')
+class call_Edit_dist():
+
+    def __init__(self, samples):
+        self.samples = samples
+
+    def evaluate(self, group_info=[], save_name='default'):
+        samples = self.samples
+        for sample in samples:
+            img_name = sample['img_id'] if sample['img_id'].endswith('.jpg') else '_'.join(
+                sample['img_id'].split('_')[:-1]
+            )
+            sample['image_name'] = img_name
+            gt = sample['norm_gt'] if sample.get('norm_gt') else sample['gt']
+            pred = sample['norm_pred'] if sample.get('norm_pred') else sample['pred']
+            upper_len = max(len(pred), len(gt))
+            sample['upper_len'] = upper_len
+            if len(pred) > 0 or len(gt) > 0:
+                edit_dist = Levenshtein.distance(pred, gt)
+                if not sample.get('metric'):
+                    sample['metric'] = {}
+                sample['metric']['Edit_dist'] = edit_dist / upper_len
+                sample['Edit_num'] = edit_dist
+
+        if isinstance(samples, list):
+            saved_samples = samples
+        else:
+            saved_samples = samples.samples
+
+        if not saved_samples:
+            return self.samples, {'Edit_dist': {'ALL_page_avg': 'NaN'}}
+
+        df = pd.DataFrame(saved_samples)
+        up_total_avg = df.groupby('image_name').apply(
+            lambda x: x['Edit_num'].sum() / x['upper_len'].sum(), include_groups=False
+        )  # page level, sum of edits divided by sum of max(gt,pred) lengths for each sample
+
+        return samples, {'Edit_dist': {'ALL_page_avg': up_total_avg.mean()}}
+
+
+@METRIC_REGISTRY.register('CDM')
+class call_CDM():
+
+    def __init__(self, samples):
+        self.samples = samples
+
+    def evaluate(self, group_info=[], save_name='default'):
+        if isinstance(self.samples, list):
+            cdm_samples = copy.deepcopy(self.samples)
+        else:
+            cdm_samples = copy.deepcopy(self.samples.samples)
+        for idx, sample in enumerate(cdm_samples):
+            sample['img_name'] = sample['img_id']
+            sample['img_id'] = str(idx)
+            sample['gt'] = sample['gt'].lstrip('$$').rstrip('$$').strip()
+            sample['pred'] = sample['pred'].split('```latex')[-1].split('```')[0]
+            sample['pred'] = sample['pred'].lstrip('$$').rstrip('$$').strip()
+
+        return self.samples, False
+
+
+class TEDS(object):
+    ''' Tree Edit Distance basead Similarity
+    '''
+
+    def __init__(self, structure_only=False, n_jobs=1, ignore_nodes=None):
+        assert isinstance(n_jobs, int) and (n_jobs >= 1), 'n_jobs must be an integer greather than 1'
+        self.structure_only = structure_only
+        self.n_jobs = n_jobs
+        self.ignore_nodes = ignore_nodes
+        self.__tokens__ = []
+
+    def tokenize(self, node):
+        ''' Tokenizes table cells
+        '''
+        self.__tokens__.append('<%s>' % node.tag)
+        if node.text is not None:
+            self.__tokens__ += list(node.text)
+        for n in node.getchildren():
+            self.tokenize(n)
+        if node.tag != 'unk':
+            self.__tokens__.append('</%s>' % node.tag)
+        if node.tag != 'td' and node.tail is not None:
+            self.__tokens__ += list(node.tail)
+
+    def load_html_tree(self, node, parent=None):
+        ''' Converts HTML tree to the format required by apted
+        '''
+        global __tokens__
+        if node.tag == 'td':
+            if self.structure_only:
+                cell = []
+            else:
+                self.__tokens__ = []
+                self.tokenize(node)
+                cell = self.__tokens__[1:-1].copy()
+            new_node = TableTree(
+                node.tag, int(node.attrib.get('colspan', '1')), int(node.attrib.get('rowspan', '1')), cell, *deque()
+            )
+        else:
+            new_node = TableTree(node.tag, None, None, None, *deque())
+        if parent is not None:
+            parent.children.append(new_node)
+        if node.tag != 'td':
+            for n in node.getchildren():
+                self.load_html_tree(n, new_node)
+        if parent is None:
+            return new_node
+
+    def evaluate(self, pred, true):
+        ''' Computes TEDS score between the prediction and the ground truth of a
+            given sample
+        '''
+        if (not pred) or (not true):
+            return 0.0
+        parser = html.HTMLParser(remove_comments=True, encoding='utf-8')
+        pred = html.fromstring(pred, parser=parser)
+        true = html.fromstring(true, parser=parser)
+        if pred.xpath('body/table') and true.xpath('body/table'):
+            pred = pred.xpath('body/table')[0]
+            true = true.xpath('body/table')[0]
+            if self.ignore_nodes:
+                etree.strip_tags(pred, *self.ignore_nodes)
+                etree.strip_tags(true, *self.ignore_nodes)
+            n_nodes_pred = len(pred.xpath('.//*'))
+            n_nodes_true = len(true.xpath('.//*'))
+            n_nodes = max(n_nodes_pred, n_nodes_true)
+            tree_pred = self.load_html_tree(pred)
+            tree_true = self.load_html_tree(true)
+            distance = APTED(tree_pred, tree_true, CustomConfig()).compute_edit_distance()
+            return 1.0 - (float(distance) / n_nodes)
+        else:
+            return 0.0
+
+    def batch_evaluate(self, pred_json, true_json):
+        ''' Computes TEDS score between the prediction and the ground truth of
+            a batch of samples
+            @params pred_json: {'FILENAME': 'HTML CODE', ...}
+            @params true_json: {'FILENAME': {'html': 'HTML CODE'}, ...}
+            @output: {'FILENAME': 'TEDS SCORE', ...}
+        '''
+        samples = true_json.keys()
+        # if self.n_jobs == 1:
+        scores = [self.evaluate(pred_json.get(filename, ''), true_json[filename]['html']) for filename in tqdm(samples)]
+        # else:
+        #     inputs = [{'pred': pred_json.get(filename, ''), 'true': true_json[filename]['html']} for filename in samples]
+        #     scores = parallel_process(inputs, self.evaluate, use_kwargs=True, n_jobs=self.n_jobs, front_num=1)
+        scores = dict(zip(samples, scores))
+        return scores
+
+
+class CustomConfig(Config):
+
+    @staticmethod
+    def maximum(*sequences):
+        """Get maximum possible value
+        """
+        return max(map(len, sequences))
+
+    def normalized_distance(self, *sequences):
+        """Get distance from 0 to 1
+        """
+        return float(Levenshtein.distance(*sequences)) / self.maximum(*sequences)
+
+    def rename(self, node1, node2):
+        """Compares attributes of trees"""
+        if (node1.tag != node2.tag) or (node1.colspan != node2.colspan) or (node1.rowspan != node2.rowspan):
+            return 1.
+        if node1.tag == 'td':
+            if node1.content or node2.content:
+                return self.normalized_distance(node1.content, node2.content)
+        return 0.
+
+
+class TableTree(Tree):
+
+    def __init__(self, tag, colspan=None, rowspan=None, content=None, *children):
+        self.tag = tag
+        self.colspan = colspan
+        self.rowspan = rowspan
+        self.content = content
+        self.children = list(children)
+
+    def bracket(self):
+        """Show tree using brackets notation"""
+        if self.tag == 'td':
+            result = '"tag": %s, "colspan": %d, "rowspan": %d, "text": %s' % \
+                     (self.tag, self.colspan, self.rowspan, self.content)
+        else:
+            result = '"tag": %s' % self.tag
+        for child in self.children:
+            result += child.bracket()
+        return '{{{}}}'.format(result)
+
+
+class recogition_end2end_base_dataset():
+
+    def __init__(self, samples):
+        img_id = 0
+        for sample in samples:
+            if not sample.get('img_id'):
+                sample['img_id'] = img_id
+            img_id += 1
+        self.samples = samples
+
+    def __getitem__(self, idx):
+        return self.samples[idx]
+
+
+class recogition_end2end_table_dataset(recogition_end2end_base_dataset):
+
+    def __init__(self, samples, table_format):
+        self.pred_table_format = table_format
+        self.samples = self.normalize_data(samples)
+
+    def normalize_data(self, samples):
+        img_id = 0
+        for sample in samples:
+            p = sample['pred']
+            r = sample['gt']
+            p = normalized_table(p, self.pred_table_format)
+            r = normalized_table(r)
+            sample['norm_gt'] = r
+            sample['norm_pred'] = p
+            sample['img_id'] = sample['img_id'] if sample.get('img_id') else img_id
+            img_id += 1
+
+        return samples

evalscope/benchmarks/omnidoc_bench/omnidoc_bench_adapter.py

@@ -0,0 +1,135 @@
+import ast
+import numpy as np
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.import_utils import check_import
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+PROMPT_TEMPLATE = r""" You are an AI assistant specialized in converting PDF images to Markdown format. Please follow these instructions for the conversion:
+
+    1. Text Processing:
+    - Accurately recognize all text content in the PDF image without guessing or inferring.
+    - Convert the recognized text into Markdown format.
+    - Maintain the original document structure, including headings, paragraphs, lists, etc.
+
+    2. Mathematical Formula Processing:
+    - Convert all mathematical formulas to LaTeX format.
+    - Enclose inline formulas with \( \). For example: This is an inline formula \( E = mc^2 \)
+    - Enclose block formulas with \\[ \\]. For example: \[ \frac{-b \pm \sqrt{b^2 - 4ac}}{2a} \]
+
+    3. Table Processing:
+    - Convert tables to HTML format.
+    - Wrap the entire table with <table> and </table>.
+
+    4. Figure Handling:
+    - Ignore figures content in the PDF image. Do not attempt to describe or convert images.
+
+    5. Output Format:
+    - Ensure the output Markdown document has a clear structure with appropriate line breaks between elements.
+    - For complex layouts, try to maintain the original document's structure and format as closely as possible.
+
+    Please strictly follow these guidelines to ensure accuracy and consistency in the conversion. Your task is to accurately convert the content of the PDF image into Markdown format without adding any extra explanations or comments.
+"""  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='omni_doc_bench',
+        pretty_name='OmniDocBench',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        """OmniDocBench is an evaluation dataset for diverse document parsing in real-world scenarios, with the following characteristics:
+- Diverse Document Types: The evaluation set contains 1355 PDF pages, covering 9 document types, 4 layout types and 3 language types. It has broad coverage including academic papers, financial reports, newspapers, textbooks, handwritten notes, etc.
+- Rich Annotations: Contains location information for 15 block-level (text paragraphs, titles, tables, etc., over 20k in total) and 4 span-level (text lines, inline formulas, superscripts/subscripts, etc., over 80k in total) document elements, as well as recognition results for each element region (text annotations, LaTeX formula annotations, tables with both LaTeX and HTML annotations). OmniDocBench also provides reading order annotations for document components. Additionally, it includes various attribute labels at page and block levels, with 5 page attribute labels, 3 text attribute labels and 6 table attribute labels.
+**The evaluation in EvalScope implements the `end2end` and `quick_match` methods from the official [OmniDocBench-v1.5 repository](https://github.com/opendatalab/OmniDocBench).**
+""",  # noqa: E501
+        dataset_id='evalscope/OmniDocBench_tsv',
+        metric_list={
+            'text_block': {
+                'metric': ['Edit_dist', 'BLEU', 'METEOR']
+            },
+            'display_formula': {
+                'metric': ['Edit_dist']
+            },
+            'table': {
+                'metric': ['TEDS', 'Edit_dist']
+            },
+            'reading_order': {
+                'metric': ['Edit_dist']
+            }
+        },
+        eval_split='train',
+        prompt_template=PROMPT_TEMPLATE,
+        extra_params={
+            'match_method': 'quick_match',
+        }
+    )
+)
+class OmniDocBenchAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_aggregation_name = False
+        self.match_method = self.extra_params.get('match_method', 'quick_match')
+
+        check_import(
+            module_name=['apted', 'distance', 'editdistance', 'Levenshtein', 'lxml', 'pylatexenc', 'bs4'],
+            package=['apted', 'distance', 'editdistance', 'Levenshtein', 'lxml', 'pylatexenc', 'BeautifulSoup4'],
+            raise_error=True,
+            feature_name='OmniDocBench'
+        )
+
+    def record_to_sample(self, record) -> Sample:
+        content_list: List[Content] = [ContentText(text=self.prompt_template)]
+        content_list.append(ContentImage(image=f'data:image/png;base64,{record["image"]}'))
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)], target='', metadata=ast.literal_eval(record['answer'])
+        )
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        # Dummy implementation to comply with the interface
+
+        score = Score(
+            prediction=original_prediction,
+            extracted_prediction=filtered_prediction,
+        )
+
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        from .end2end_eval import End2EndEvaluator
+
+        if not sample_scores:
+            return []
+
+        predictions = [s.score.prediction for s in sample_scores]
+        references = [s.sample_metadata for s in sample_scores]
+
+        evaluator = End2EndEvaluator(
+            prediction=predictions,
+            reference=references,
+            metrics=self.metric_list,
+            match_method=self.match_method,
+        )
+        agg_results = evaluator.score()
+
+        agg_scores = []
+        for metric_name, agg_result in agg_results.items():
+            if agg_result is not np.nan:
+                agg_score = AggScore(
+                    score=agg_result,
+                    metric_name=metric_name,
+                    num=len(sample_scores),
+                )
+                agg_scores.append(agg_score)
+
+        return agg_scores