evalscope 1.0.2__py3-none-any.whl → 1.1.1__py3-none-any.whl

evalscope/benchmarks/ner/mit_movie_trivia_adapter.py

@@ -0,0 +1,74 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'The MIT-Movie-Trivia dataset, originally created for slot filling, is modified by '
+    'ignoring some slot types (e.g. genre, rating) and merging others (e.g. director '
+    'and actor in person, and song and movie title in title) in order to keep '
+    'consistent named entity types across all datasets.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mit-movie-trivia',
+        pretty_name='MIT-Movie-Trivia',
+        dataset_id='extraordinarylab/mit-movie-trivia',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class MITMovieTriviaAdapter(NERAdapter):
+    """
+    Adapter for the MIT-Movie-Trivia Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the MIT-Movie-Trivia dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define MIT-Movie-Trivia-specific entity mappings
+        self.entity_type_map = {
+            'ACTOR': 'actor',
+            'AWARD': 'award',
+            'CHARACTER_NAME': 'character_name',
+            'DIRECTOR': 'director',
+            'GENRE': 'genre',
+            'OPINION': 'opinion',
+            'ORIGIN': 'origin',
+            'PLOT': 'plot',
+            'QUOTE': 'quote',
+            'RELATIONSHIP': 'relationship',
+            'SOUNDTRACK': 'soundtrack',
+            'YEAR': 'year'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'ACTOR': 'The name of an actor or actress starring in the movie.',
+            'AWARD': 'An award the movie won or was nominated for.',
+            'CHARACTER_NAME': 'The name of a character in the movie.',
+            'DIRECTOR': 'The name of the person who directed the movie.',
+            'GENRE': 'The category or style of the movie.',
+            'OPINION': 'A subjective review or personal opinion about the movie.',
+            'ORIGIN': 'The source material or basis for the movie.',
+            'PLOT': 'A description or summary of the movie\'s storyline.',
+            'QUOTE': 'A memorable line or phrase spoken in the movie.',
+            'RELATIONSHIP': 'The connection or relationship between characters.',
+            'SOUNDTRACK': 'The music or a specific song from the movie.',
+            'YEAR': 'The release year of the movie.'
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/mit_restaurant_adapter.py

@@ -0,0 +1,66 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'The MIT-Restaurant dataset is a collection of restaurant review text specifically '
+    'curated for training and testing Natural Language Processing (NLP) models, '
+    'particularly for Named Entity Recognition (NER). It contains sentences from real '
+    'reviews, along with corresponding labels in the BIO format.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mit-restaurant',
+        pretty_name='MIT-Restaurant',
+        dataset_id='extraordinarylab/mit-restaurant',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class MITRestaurantAdapter(NERAdapter):
+    """
+    Adapter for the MIT-Restaurant Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the MIT-Restaurant dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define MIT-Restaurant-specific entity mappings
+        self.entity_type_map = {
+            'AMENITY': 'amenity',
+            'CUISINE': 'cuisine',
+            'DISH': 'dish',
+            'HOURS': 'hours',
+            'LOCATION': 'location',
+            'PRICE': 'price',
+            'RATING': 'rating',
+            'RESTAURANT_NAME': 'restaurant_name'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'AMENITY': 'A feature or service offered by the restaurant.',
+            'CUISINE': 'The type of food a restaurant serves.',
+            'DISH': 'A specific food or drink item.',
+            'HOURS': 'The operating hours of a restaurant.',
+            'LOCATION': 'The address or general location of a restaurant.',
+            'PRICE': 'The price range of a restaurant.',
+            'RATING': 'A rating or review of the restaurant.',
+            'RESTAURANT_NAME': 'The name of a restaurant.',
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/ontonotes5_adapter.py

@@ -0,0 +1,87 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'OntoNotes Release 5.0 is a large, multilingual corpus containing text in English, '
+    'Chinese, and Arabic across various genres like news, weblogs, and broadcast '
+    'conversations. It is richly annotated with multiple layers of linguistic information, '
+    'including syntax, predicate-argument structure, word sense, named entities, and '
+    'coreference to support research and development in natural language processing.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ontonotes5',
+        pretty_name='OntoNotes5',
+        dataset_id='extraordinarylab/ontonotes5',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class OntoNotes5Adapter(NERAdapter):
+    """
+    Adapter for the OntoNotes5 Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the OntoNotes5 dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define OntoNotes5-specific entity mappings
+        self.entity_type_map = {
+            'CARDINAL': 'cardinal',
+            'DATE': 'date',
+            'EVENT': 'event',
+            'FAC': 'facility',
+            'GPE': 'geopolitical_entity',
+            'LANGUAGE': 'language',
+            'LAW': 'law',
+            'LOC': 'location',
+            'MONEY': 'money',
+            'NORP': 'nationalities_or_religious_or_political_groups',
+            'ORDINAL': 'ordinal',
+            'ORG': 'organization',
+            'PERCENT': 'percent',
+            'PERSON': 'person',
+            'PRODUCT': 'product',
+            'QUANTITY': 'quantity',
+            'TIME': 'time',
+            'WORK_OF_ART': 'work_of_art'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'PERSON': 'People, including fictional',
+            'NORP': 'Nationalities or religious or political groups',
+            'FAC': 'Buildings, airports, highways, bridges, etc.',
+            'ORG': 'Companies, agencies, institutions, etc.',
+            'GPE': 'Countries, cities, states',
+            'LOC': 'Non-GPE locations, mountain ranges, bodies of water',
+            'PRODUCT': 'Vehicles, weapons, foods, etc. (Not services)',
+            'EVENT': 'Named hurricanes, battles, wars, sports events, etc.',
+            'WORK_OF_ART': 'Titles of books, songs, etc.',
+            'LAW': 'Named documents made into laws',
+            'LANGUAGE': 'Any named language',
+            'DATE': 'Absolute or relative dates or periods',
+            'TIME': 'Times smaller than a day',
+            'PERCENT': 'Percentage (including "%")',
+            'MONEY': 'Monetary values, including unit',
+            'QUANTITY': 'Measurements, as of weight or distance',
+            'ORDINAL': '"first", "second"',
+            'CARDINAL': 'Numerals that do not fall under another type'
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/wnut2017_adapter.py

@@ -0,0 +1,61 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'The WNUT2017 dataset is a collection of user-generated text from various social '
+    'media platforms, like Twitter and YouTube, specifically designed for a named-entity '
+    'recognition task.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='wnut2017',
+        pretty_name='WNUT2017',
+        dataset_id='extraordinarylab/wnut2017',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class WNUT2017Adapter(NERAdapter):
+    """
+    Adapter for the WNUT2017 Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the WNUT2017 dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define WNUT2017-specific entity mappings
+        self.entity_type_map = {
+            'CORPORATION': 'corporation',
+            'CREATIVE-WORK': 'creative_work',
+            'GROUP': 'group',
+            'LOCATION': 'location',
+            'PERSON': 'person',
+            'PRODUCT': 'product'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'CORPORATION': 'Named companies, businesses, agencies, and other institutions.',
+            'CREATIVE-WORK': 'Named books, songs, movies, paintings, and other works of art.',
+            'GROUP': 'Named groups of people, such as sports teams, bands, or political groups.',
+            'LOCATION': 'Named geographical locations, such as cities, countries, and natural landmarks.',
+            'PERSON': 'Named individuals, including both real and fictional people.',
+            'PRODUCT': 'Named commercial products, including vehicles, software, and other goods.'
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ocr_bench/ocr_bench_adapter.py

@@ -0,0 +1,101 @@
+import json
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric.scorer import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'Regular Text Recognition', 'Irregular Text Recognition', 'Artistic Text Recognition', 'Handwriting Recognition',
+    'Digit String Recognition', 'Non-Semantic Text Recognition', 'Scene Text-centric VQA', 'Doc-oriented VQA',
+    'Key Information Extraction', 'Handwritten Mathematical Expression Recognition'
+]
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ocr_bench',
+        pretty_name='OCRBench',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'OCRBench is a comprehensive evaluation benchmark designed to assess the OCR capabilities of Large Multimodal Models. It comprises five components: Text Recognition, SceneText-Centric VQA, Document-Oriented VQA, Key Information Extraction, and Handwritten Mathematical Expression Recognition. The benchmark includes 1000 question-answer pairs, and all the answers undergo manual verification and correction to ensure a more precise evaluation.',  # noqa: E501
+        dataset_id='evalscope/OCRBench',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template='{question}',
+    )
+)
+class OCRBenchAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_aggregation_name = False
+        self.reformat_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+
+        input_text = self.prompt_template.format(question=record['question'])
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='jpeg', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=json.dumps(record.get('answer'), ensure_ascii=False),  # answers is a list
+            subset_key=record.get('question_type'),
+            metadata={
+                'dataset': record.get('dataset'),
+                'question_type': record.get('question_type'),
+            }
+        )
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        pred = filtered_prediction.lower().strip()
+        gt_ans = json.loads(reference)
+        dataset_name = task_state.metadata['dataset']
+
+        score_value = 0
+        if dataset_name == 'HME100k':
+            if isinstance(gt_ans, list):
+                for j in range(len(gt_ans)):
+                    answer = gt_ans[j].strip().replace('\n', ' ').replace(' ', '')
+                    predict = pred.strip().replace('\n', ' ').replace(' ', '')
+                    if answer in predict:
+                        score_value = 1
+            else:
+                answer = gt_ans.strip().replace('\n', ' ').replace(' ', '')
+                predict = pred.strip().replace('\n', ' ').replace(' ', '')
+                if answer in predict:
+                    score_value = 1
+        else:
+            if isinstance(gt_ans, list):
+                for j in range(len(gt_ans)):
+                    answer = gt_ans[j].lower().strip().replace('\n', ' ')
+                    predict = pred.lower().strip().replace('\n', ' ')
+                    if answer in predict:
+                        score_value = 1
+            else:
+                answer = gt_ans.lower().strip().replace('\n', ' ')
+                predict = pred.lower().strip().replace('\n', ' ')
+                if answer in predict:
+                    score_value = 1
+        score.value = {'acc': score_value}
+        return score

evalscope/benchmarks/ocr_bench_v2/IoUscore_metric.py

@@ -0,0 +1,87 @@
+# flake8: noqa
+import ast
+import re
+
+from .vqa_metric import vqa_evaluation
+
+
+def calculate_iou(box1, box2):
+    try:
+        box1 = [int(coordinate) for coordinate in box1]
+        box2 = [int(coordinate) for coordinate in box2]
+    except:
+        return 0
+
+    x1_inter = max(box1[0], box2[0])
+    y1_inter = max(box1[1], box2[1])
+    x2_inter = min(box1[2], box2[2])
+    y2_inter = min(box1[3], box2[3])
+
+    inter_area = max(0, x2_inter - x1_inter) * max(0, y2_inter - y1_inter)
+
+    box1_area = (box1[2] - box1[0]) * (box1[3] - box1[1])
+    box2_area = (box2[2] - box2[0]) * (box2[3] - box2[1])
+
+    union_area = box1_area + box2_area - inter_area
+
+    iou = inter_area / union_area if union_area != 0 else 0
+
+    return iou
+
+
+def vqa_with_position_evaluation(predict, img_metas):
+    score_content, score_bbox = 0.0, 0.0
+    if 'answer' in predict.keys():
+        score_content = vqa_evaluation(predict['answer'], img_metas['answers'])
+    if 'bbox' in predict.keys():
+        gt_bbox = img_metas['bbox']
+        try:
+            predict_bbox_list = ast.literal_eval(predict['bbox'])
+            score_bbox = calculate_iou(predict_bbox_list, gt_bbox)
+        except:
+            score_bbox = 0
+    return 0.5 * score_content + 0.5 * score_bbox
+
+
+def extract_coordinates(text):
+    # Regex pattern to match coordinates in either (x1, y1, x2, y2) or [x1, y1, x2, y2] format
+
+    pattern = r'[\(\[]\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*,\s*(\d+)\s*[\)\]]'
+
+    matches = list(re.finditer(pattern, text))
+    coords_list = []
+    coords_set = set()
+    for match in matches:
+        x1, y1, x2, y2 = map(int, match.groups())
+
+        if all(0 <= n <= 1000 for n in [x1, y1, x2, y2]):
+            coords = (x1, y1, x2, y2)
+
+            if coords in coords_set:
+                coords_list = [c for c in coords_list if c != coords]
+
+            coords_list.append(coords)
+            coords_set.add(coords)
+    if coords_list:
+        last_coords = coords_list[-1]
+        return list(last_coords)
+    else:
+        return None
+
+
+if __name__ == '__main__':
+    print('Example for Text Grounding task.')
+    box1 = [50, 50, 150, 150]
+    box2 = [60, 60, 140, 140]
+    iou_score = calculate_iou(box1, box2)
+    print(f'IoU score: {iou_score}')
+
+    print('Example for VQA with position task.')
+    pred = {'content': 'The content is Hello Buddies', 'bbox': box1}
+    gt = {'content': 'Hello Buddies', 'bbox': box2}
+
+    vqa_score = vqa_evaluation(pred['content'], gt['content'])
+    iou_score = calculate_iou(pred['bbox'], gt['bbox'])
+
+    print(f'VQA score: {vqa_score}')
+    print(f'IoU score: {iou_score}')