evalscope 1.1.0__py3-none-any.whl → 1.1.1__py3-none-any.whl

evalscope/benchmarks/ner/mit_movie_trivia_adapter.py

@@ -0,0 +1,74 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'The MIT-Movie-Trivia dataset, originally created for slot filling, is modified by '
+    'ignoring some slot types (e.g. genre, rating) and merging others (e.g. director '
+    'and actor in person, and song and movie title in title) in order to keep '
+    'consistent named entity types across all datasets.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mit-movie-trivia',
+        pretty_name='MIT-Movie-Trivia',
+        dataset_id='extraordinarylab/mit-movie-trivia',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class MITMovieTriviaAdapter(NERAdapter):
+    """
+    Adapter for the MIT-Movie-Trivia Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the MIT-Movie-Trivia dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define MIT-Movie-Trivia-specific entity mappings
+        self.entity_type_map = {
+            'ACTOR': 'actor',
+            'AWARD': 'award',
+            'CHARACTER_NAME': 'character_name',
+            'DIRECTOR': 'director',
+            'GENRE': 'genre',
+            'OPINION': 'opinion',
+            'ORIGIN': 'origin',
+            'PLOT': 'plot',
+            'QUOTE': 'quote',
+            'RELATIONSHIP': 'relationship',
+            'SOUNDTRACK': 'soundtrack',
+            'YEAR': 'year'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'ACTOR': 'The name of an actor or actress starring in the movie.',
+            'AWARD': 'An award the movie won or was nominated for.',
+            'CHARACTER_NAME': 'The name of a character in the movie.',
+            'DIRECTOR': 'The name of the person who directed the movie.',
+            'GENRE': 'The category or style of the movie.',
+            'OPINION': 'A subjective review or personal opinion about the movie.',
+            'ORIGIN': 'The source material or basis for the movie.',
+            'PLOT': 'A description or summary of the movie\'s storyline.',
+            'QUOTE': 'A memorable line or phrase spoken in the movie.',
+            'RELATIONSHIP': 'The connection or relationship between characters.',
+            'SOUNDTRACK': 'The music or a specific song from the movie.',
+            'YEAR': 'The release year of the movie.'
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/mit_restaurant_adapter.py

@@ -0,0 +1,66 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'The MIT-Restaurant dataset is a collection of restaurant review text specifically '
+    'curated for training and testing Natural Language Processing (NLP) models, '
+    'particularly for Named Entity Recognition (NER). It contains sentences from real '
+    'reviews, along with corresponding labels in the BIO format.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mit-restaurant',
+        pretty_name='MIT-Restaurant',
+        dataset_id='extraordinarylab/mit-restaurant',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class MITRestaurantAdapter(NERAdapter):
+    """
+    Adapter for the MIT-Restaurant Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the MIT-Restaurant dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define MIT-Restaurant-specific entity mappings
+        self.entity_type_map = {
+            'AMENITY': 'amenity',
+            'CUISINE': 'cuisine',
+            'DISH': 'dish',
+            'HOURS': 'hours',
+            'LOCATION': 'location',
+            'PRICE': 'price',
+            'RATING': 'rating',
+            'RESTAURANT_NAME': 'restaurant_name'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'AMENITY': 'A feature or service offered by the restaurant.',
+            'CUISINE': 'The type of food a restaurant serves.',
+            'DISH': 'A specific food or drink item.',
+            'HOURS': 'The operating hours of a restaurant.',
+            'LOCATION': 'The address or general location of a restaurant.',
+            'PRICE': 'The price range of a restaurant.',
+            'RATING': 'A rating or review of the restaurant.',
+            'RESTAURANT_NAME': 'The name of a restaurant.',
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/ontonotes5_adapter.py

@@ -0,0 +1,87 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'OntoNotes Release 5.0 is a large, multilingual corpus containing text in English, '
+    'Chinese, and Arabic across various genres like news, weblogs, and broadcast '
+    'conversations. It is richly annotated with multiple layers of linguistic information, '
+    'including syntax, predicate-argument structure, word sense, named entities, and '
+    'coreference to support research and development in natural language processing.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ontonotes5',
+        pretty_name='OntoNotes5',
+        dataset_id='extraordinarylab/ontonotes5',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class OntoNotes5Adapter(NERAdapter):
+    """
+    Adapter for the OntoNotes5 Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the OntoNotes5 dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define OntoNotes5-specific entity mappings
+        self.entity_type_map = {
+            'CARDINAL': 'cardinal',
+            'DATE': 'date',
+            'EVENT': 'event',
+            'FAC': 'facility',
+            'GPE': 'geopolitical_entity',
+            'LANGUAGE': 'language',
+            'LAW': 'law',
+            'LOC': 'location',
+            'MONEY': 'money',
+            'NORP': 'nationalities_or_religious_or_political_groups',
+            'ORDINAL': 'ordinal',
+            'ORG': 'organization',
+            'PERCENT': 'percent',
+            'PERSON': 'person',
+            'PRODUCT': 'product',
+            'QUANTITY': 'quantity',
+            'TIME': 'time',
+            'WORK_OF_ART': 'work_of_art'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'PERSON': 'People, including fictional',
+            'NORP': 'Nationalities or religious or political groups',
+            'FAC': 'Buildings, airports, highways, bridges, etc.',
+            'ORG': 'Companies, agencies, institutions, etc.',
+            'GPE': 'Countries, cities, states',
+            'LOC': 'Non-GPE locations, mountain ranges, bodies of water',
+            'PRODUCT': 'Vehicles, weapons, foods, etc. (Not services)',
+            'EVENT': 'Named hurricanes, battles, wars, sports events, etc.',
+            'WORK_OF_ART': 'Titles of books, songs, etc.',
+            'LAW': 'Named documents made into laws',
+            'LANGUAGE': 'Any named language',
+            'DATE': 'Absolute or relative dates or periods',
+            'TIME': 'Times smaller than a day',
+            'PERCENT': 'Percentage (including "%")',
+            'MONEY': 'Monetary values, including unit',
+            'QUANTITY': 'Measurements, as of weight or distance',
+            'ORDINAL': '"first", "second"',
+            'CARDINAL': 'Numerals that do not fall under another type'
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/wnut2017_adapter.py

@@ -0,0 +1,61 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'The WNUT2017 dataset is a collection of user-generated text from various social '
+    'media platforms, like Twitter and YouTube, specifically designed for a named-entity '
+    'recognition task.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='wnut2017',
+        pretty_name='WNUT2017',
+        dataset_id='extraordinarylab/wnut2017',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class WNUT2017Adapter(NERAdapter):
+    """
+    Adapter for the WNUT2017 Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the WNUT2017 dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define WNUT2017-specific entity mappings
+        self.entity_type_map = {
+            'CORPORATION': 'corporation',
+            'CREATIVE-WORK': 'creative_work',
+            'GROUP': 'group',
+            'LOCATION': 'location',
+            'PERSON': 'person',
+            'PRODUCT': 'product'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'CORPORATION': 'Named companies, businesses, agencies, and other institutions.',
+            'CREATIVE-WORK': 'Named books, songs, movies, paintings, and other works of art.',
+            'GROUP': 'Named groups of people, such as sports teams, bands, or political groups.',
+            'LOCATION': 'Named geographical locations, such as cities, countries, and natural landmarks.',
+            'PERSON': 'Named individuals, including both real and fictional people.',
+            'PRODUCT': 'Named commercial products, including vehicles, software, and other goods.'
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ocr_bench_v2/utils.py

@@ -1,5 +1,6 @@
 # flake8: noqa
 import ast
+import os
 import re
 
 from .IoUscore_metric import calculate_iou, extract_coordinates, vqa_with_position_evaluation

evalscope/benchmarks/omnidoc_bench/end2end_eval.py

@@ -0,0 +1,349 @@
+# flake8: noqa
+import numpy as np
+import os
+import sys
+from collections import defaultdict
+from typing import Dict, List
+
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+
+class End2EndEvaluator():
+
+    def __init__(
+        self,
+        prediction: List,
+        reference: List,
+        metrics: Dict,
+        match_method: str = 'quick_match',
+        filter_types: dict = None
+    ):
+
+        self.match_method = match_method
+        self.references = reference
+        self.predictions = prediction
+        self.dafault_metircs_dict = metrics
+
+        filtered_gt_samples = []
+        if filter_types:
+            for gt_sample in self.references:
+                select_flag = True
+                for k, v in filter_types.items():
+                    if gt_sample['page_info']['page_attribute'][k] != v:
+                        select_flag = False
+                if select_flag:
+                    filtered_gt_samples.append(gt_sample)
+        else:
+            filtered_gt_samples = self.references  #[{},{},{}]
+        self.references = filtered_gt_samples
+
+    def score(self) -> dict:
+        samples = self.get_matched_elements(self.references, self.predictions)
+        metrics = self.process_generated_metric_results(samples)
+        return metrics
+
+    def get_page_elements(self, selected_annos):
+        saved_element_dict = defaultdict(list)
+        related_truncated = []
+        truncated_all = {}
+        for relation in selected_annos['extra']['relation']:  # Handle truncated text issues
+            if relation['relation_type'] == 'truncated':
+                truncated_all[relation['source_anno_id']] = ''
+                truncated_all[relation['target_anno_id']] = ''
+                exist_flag = False
+                for merge_list in related_truncated:
+                    if relation['source_anno_id'] in merge_list or relation[
+                        'target_anno_id'] in merge_list:  # Consider cases where three text blocks may need to be merged
+                        merge_list.append(relation['source_anno_id'])
+                        merge_list.append(relation['target_anno_id'])
+                        exist_flag = True
+                if not exist_flag:
+                    related_truncated.append([relation['source_anno_id'], relation['target_anno_id']])
+
+        for item in selected_annos['layout_dets']:
+            if item['anno_id'] not in truncated_all.keys():
+                saved_element_dict[item['category_type']].append(item)
+            else:
+                truncated_all[item['anno_id']] = item
+
+        for merge_list in related_truncated:
+            text_block_list = [truncated_all[key] for key in merge_list]
+            sorted_block = sorted(text_block_list, key=lambda x: x['order'])
+            text = ''
+            for block in sorted_block:
+                text += block['text']
+            merged_block = {
+                'category_type': sorted_block[0]['category_type'],  # Directly use information from the first block
+                'order': sorted_block[0]['order'],
+                'anno_id': sorted_block[0]['anno_id'],
+                'text': text,
+                'merge_list': sorted_block
+            }
+            saved_element_dict[sorted_block[0]['category_type']].append(merged_block)
+
+        return saved_element_dict
+
+    def get_page_elements_list(self, gt_page_elements, category_list):
+        element_list = []
+        for category_type in category_list:
+            if gt_page_elements.get(category_type):
+                element_list.extend(gt_page_elements[category_type])
+        return element_list
+
+    def get_sorted_text_list(self, selected_annos):
+        # txt_type: text, latex, html
+        text_list = []
+        for item in selected_annos:
+            if item.get('order'):
+                order = item['order']
+            else:
+                order = 0
+            # 【txt_type,selecte_annos]
+            text_list.append((order, item))
+        sorted_text_list = sorted(text_list, key=lambda x: x[0])
+        return [_[1] for _ in sorted_text_list]
+
+    def filtered_out_ignore(self, items, ignore_category_list):
+        filted_items = []
+        for item in items:
+            if item['gt_category_type'] not in ignore_category_list:
+                filted_items.append(item)
+        return filted_items
+
+    def get_order_paired(self, order_match_s, img_name):
+        matched = [(item['gt_position'], item['pred_position'])
+                   for item in order_match_s
+                   if (item['gt_position'] != [''] and item['pred_position'] != '')]
+        gt_idx_all = [item['gt_position'] for item in order_match_s if (item['gt_position'] != [''])]
+        read_order_pred = [i[0] for i in sorted(matched, key=lambda x: x[1])]
+        read_order_gt = sum(gt_idx_all, [])  # Convert to one-dimensional list
+        read_order_gt = [x for x in read_order_gt if x]
+        gt = sorted(read_order_gt)
+        pred = sum(read_order_pred, [])
+        pred = [x for x in pred if x]
+        if len(pred) > 0 or len(gt) > 0:
+            import Levenshtein
+            edit = Levenshtein.distance(gt, pred) / max(len(pred), len(gt))
+            return {'gt': gt, 'pred': pred, 'img_id': img_name, 'edit': edit}
+        else:
+            return {}  # If both GT and pred are empty for the page, return empty
+
+    def formula_format(self, formula_matches, img_name):
+        # formated_list = []
+        for i, item in enumerate(formula_matches):
+            item['img_id'] = img_name + '_' + str(i)
+        return formula_matches
+
+    def get_matched_elements(self, references: list, predictions: list) -> dict:
+        from .metrics import recogition_end2end_base_dataset, recogition_end2end_table_dataset
+
+        plain_text_match = []
+        display_formula_match = []
+        html_table_match = []
+        latex_table_match = []
+        order_match = []
+
+        for i, sample in enumerate(references):
+            img_name = os.path.basename(sample['page_info']['image_path'])
+            pred_content = predictions[i]
+            result = self.process_get_matched_elements(sample, pred_content, img_name)
+            [
+                plain_text_match_clean, formated_display_formula, latex_table_match_s, html_table_match_s,
+                order_match_single
+            ] = result
+
+            if order_match_single:
+                order_match.append(order_match_single)
+            if plain_text_match_clean:
+                plain_text_match.extend(plain_text_match_clean)
+            if formated_display_formula:
+                display_formula_match.extend(formated_display_formula)
+            if latex_table_match_s:
+                latex_table_match.extend(latex_table_match_s)
+            if html_table_match_s:
+                html_table_match.extend(html_table_match_s)
+
+        if len(latex_table_match) > len(html_table_match):
+            table_match = latex_table_match
+            table_format = 'latex'
+        else:
+            table_match = html_table_match
+            table_format = 'html'
+
+        matched_samples_all = {
+            'text_block': recogition_end2end_base_dataset(plain_text_match),
+            'display_formula': recogition_end2end_base_dataset(display_formula_match),
+            'table': recogition_end2end_table_dataset(table_match, table_format),
+            'reading_order': recogition_end2end_base_dataset(order_match)
+        }
+
+        return matched_samples_all
+
+    def process_get_matched_elements(self, sample, pred_content, img_name):
+        from func_timeout import FunctionTimedOut, func_timeout
+
+        from .utils import match_gt2pred_no_split, match_gt2pred_quick, match_gt2pred_simple, md_tex_filter
+
+        if self.match_method == 'simple_match':  # add match choice
+            match_gt2pred = match_gt2pred_simple
+        elif self.match_method == 'quick_match':
+            match_gt2pred = match_gt2pred_quick
+        elif self.match_method == 'no_split':
+            match_gt2pred = match_gt2pred_no_split
+        else:
+            match_gt2pred = match_gt2pred_quick
+
+        pred_dataset = md_tex_filter(pred_content)
+        gt_page_elements = self.get_page_elements(sample)
+
+        text_all = self.get_page_elements_list(
+            gt_page_elements, [
+                'text_block', 'title', 'code_txt', 'code_txt_caption', 'reference', 'equation_caption',
+                'figure_caption', 'figure_footnote', 'table_caption', 'table_footnote', 'code_algorithm',
+                'code_algorithm_caption', 'header', 'footer', 'page_footnote', 'page_number'
+            ]
+        )
+
+        display_formula_match_s = []
+        plain_text_match_clean = []
+        latex_table_match_s = []
+        html_table_match_s = []
+        order_match_single = []
+        if text_all:
+            gt_text_list = self.get_sorted_text_list(text_all)
+            try:
+                plain_text_match_s = func_timeout(
+                    30, match_gt2pred, args=(gt_text_list, pred_dataset['text_all'], 'text', img_name)
+                )
+            except FunctionTimedOut as e:
+                logger.warning(f'Time out for plain text match of {img_name}, match_gt2pred_simple will be used.')
+                plain_text_match_s = match_gt2pred_simple(gt_text_list, pred_dataset['text_all'], 'text', img_name)
+                logger.error(str(e))
+                raise e
+
+            if not plain_text_match_s:
+                logger.warning(f'No text match of {img_name}. The plain text match will be empty.')
+            else:
+                plain_text_match_clean = self.filtered_out_ignore(
+                    plain_text_match_s, [
+                        'figure_caption', 'figure_footnote', 'table_caption', 'table_footnote', 'code_algorithm',
+                        'code_algorithm_caption', 'header', 'footer', 'page_footnote', 'page_number', 'equation_caption'
+                    ]
+                )
+
+        if gt_page_elements.get('equation_isolated'):
+            gt_display_list = self.get_sorted_text_list(gt_page_elements['equation_isolated'])
+            display_formula_match_s = match_gt2pred(
+                gt_display_list, pred_dataset['equation_isolated'], 'formula', img_name
+            )
+            display_formula_match_s = [x for x in display_formula_match_s if x['gt_idx'] != ['']]
+            if not display_formula_match_s:
+                logger.warning(f'No display_formula_match of {img_name}. The display_formula_match will be empty.')
+
+        if gt_page_elements.get('table'):
+            gt_table_list = self.get_sorted_text_list(gt_page_elements['table'])
+            if pred_dataset['latex_table']:
+                latex_table_match_s = match_gt2pred_simple(
+                    gt_table_list, pred_dataset['latex_table'], 'latex_table', img_name
+                )
+                latex_table_match_s = [x for x in latex_table_match_s if x['gt_idx'] != ['']]
+            if pred_dataset['html_table']:
+                html_table_match_s = match_gt2pred_simple(
+                    gt_table_list, pred_dataset['html_table'], 'html_table', img_name
+                )
+                html_table_match_s = [x for x in html_table_match_s if x['gt_idx'] != ['']]
+            else:
+                html_table_match_s = match_gt2pred_simple(gt_table_list, [], 'html_table', img_name)
+                html_table_match_s = [x for x in html_table_match_s if x['gt_idx'] != ['']]
+
+        order_match_s = plain_text_match_clean
+        if order_match_s:
+            order_match_single = self.get_order_paired(order_match_s, img_name)
+
+        return [
+            plain_text_match_clean, display_formula_match_s, latex_table_match_s, html_table_match_s, order_match_single
+        ]
+
+    def process_generated_metric_results(self, samples, save_name: str = 'end2end_quick_match'):
+        from .metrics import METRIC_REGISTRY, get_full_labels_results, get_page_split, show_result
+
+        result_all = {}
+        page_info = {}
+        metircs_dict = self.dafault_metircs_dict
+        pages = self.references  #gt_samples list
+
+        for page in pages:
+            img_path = os.path.basename(page['page_info']['image_path'])
+            page_info[img_path] = page['page_info']['page_attribute']
+
+        for element in metircs_dict.keys():
+
+            result = {}
+            group_info = metircs_dict[element].get('group', [])
+            # samples = samples.get(element) ##
+            cur_samples = samples[element]
+
+            for metric in metircs_dict[element]['metric']:
+                metric_val = METRIC_REGISTRY.get(metric)
+
+                cur_samples, result_s = metric_val(cur_samples).evaluate(group_info, f'{save_name}_{element}')
+                if result_s:
+                    result.update(result_s)
+
+            if result:
+                logger.info(f'{element}')
+                show_result(result)
+            result_all[element] = {}
+
+            group_result = get_full_labels_results(cur_samples)
+            page_result = get_page_split(cur_samples, page_info)
+
+            result_all[element] = {'all': result, 'group': group_result, 'page': page_result}
+
+        save_dict = {}
+        en_overall = []
+        ch_overall = []
+        for category_type, metric in [('text_block', 'Edit_dist'), ('display_formula', 'Edit_dist'),
+                                      ('display_formula', 'CDM'), ('table', 'TEDS'), ('table', 'Edit_dist'),
+                                      ('reading_order', 'Edit_dist')]:
+            if metric == 'TEDS':
+                if category_type in result_all and 'page' in result_all[category_type] and metric in result_all[
+                    category_type]['page']:
+                    save_dict[category_type + '_' + metric
+                              + '_EN'] = result_all[category_type]['page'][metric]['language: english']
+                    save_dict[category_type + '_' + metric
+                              + '_CH'] = result_all[category_type]['page'][metric]['language: simplified_chinese']
+                else:
+                    save_dict[category_type + '_' + metric + '_EN'] = np.nan
+                    save_dict[category_type + '_' + metric + '_CH'] = np.nan
+            else:
+                if category_type in result_all and 'page' in result_all[category_type] and metric in result_all[
+                    category_type]['page']:
+                    save_dict[category_type + '_' + metric
+                              + '_EN'] = result_all[category_type]['page'][metric].get('language: english', np.nan)
+                    save_dict[category_type + '_' + metric + '_CH'] = result_all[category_type]['page'][metric].get(
+                        'language: simplified_chinese', np.nan
+                    )
+                else:
+                    save_dict[category_type + '_' + metric + '_EN'] = np.nan
+                    save_dict[category_type + '_' + metric + '_CH'] = np.nan
+
+            if metric == 'Edit_dist':
+                if category_type in result_all and 'page' in result_all[category_type] and metric in result_all[
+                    category_type]['page']:
+                    en_overall.append(result_all[category_type]['page'][metric].get('language: english', np.nan))
+                    ch_overall.append(
+                        result_all[category_type]['page'][metric].get('language: simplified_chinese', np.nan)
+                    )
+                else:
+                    en_overall.append(np.nan)
+                    ch_overall.append(np.nan)
+
+        en_overall_filtered = [x for x in en_overall if not np.isnan(x)]
+        ch_overall_filtered = [x for x in ch_overall if not np.isnan(x)]
+        save_dict['overall_EN'] = sum(en_overall_filtered) / len(en_overall_filtered) if en_overall_filtered else np.nan
+        save_dict['overall_CH'] = sum(ch_overall_filtered) / len(ch_overall_filtered) if ch_overall_filtered else np.nan
+
+        return save_dict