evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/ner/cross_ner_adapter.py

@@ -0,0 +1,120 @@
+from typing import Any, Dict, List, Set, Tuple
+
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.benchmarks.ner.cross_ner_entities import ai, literature, music, politics, science
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE, create_target_text
+
+DESCRIPTION = (
+    'CrossNER is a fully-labelled collected of named entity recognition (NER) data '
+    'spanning over five diverse domains (AI, Literature, Music, Politics, Science).'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='cross_ner',
+        pretty_name='CrossNER',
+        dataset_id='extraordinarylab/cross-ner',
+        subset_list=['ai', 'literature', 'music', 'politics', 'science'],
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class CrossNERAdapter(NERAdapter):
+    """
+    Adapter for the CrossNER Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the CrossNER dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define CrossNER-specific entity mappings
+        self.entity_type_map = {}
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {}
+
+    def setup_entity_mappings(self):
+        """
+        Setup entity mappings and descriptions for prompt formatting.
+        This should be called after entity_type_map and entity_descriptions are defined.
+        """
+        if self.current_subset_name == 'ai':
+            self.entity_type_map, self.entity_descriptions = ai.get_entity_mappings()
+        elif self.current_subset_name == 'literature':
+            self.entity_type_map, self.entity_descriptions = literature.get_entity_mappings()
+        elif self.current_subset_name == 'music':
+            self.entity_type_map, self.entity_descriptions = music.get_entity_mappings()
+        elif self.current_subset_name == 'politics':
+            self.entity_type_map, self.entity_descriptions = politics.get_entity_mappings()
+        elif self.current_subset_name == 'science':
+            self.entity_type_map, self.entity_descriptions = science.get_entity_mappings()
+
+        # Reverse mapping for converting back from prediction to evaluation
+        self.reverse_entity_map = {v.lower(): k for k, v in self.entity_type_map.items()}
+
+        # Create list of tags for prompt formatting
+        self.entity_list = [f'<{ent.lower()}>' for ent in self.entity_type_map.values()]
+
+        # Create description of entities for prompt
+        self.entities_description = ', '.join([
+            f'{self.entity_type_map[tag]} ({self.entity_descriptions[tag]})' for tag in self.entity_type_map
+        ])
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a record with tokens and NER tags into a Sample.
+        Creates both the raw text input and annotated text target.
+        """
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()
+
+        tokens: List[str] = record['tokens']
+        ner_tags: List[str] = record['ner_tags']
+
+        # Create the input text by joining tokens
+        input_text = ' '.join(tokens)
+
+        # Process tokens and tags to create annotated target text
+        target_text = create_target_text(tokens, ner_tags, self.entity_type_map)
+
+        # Store tokens and tags in metadata for evaluation
+        metadata = {'tokens': tokens, 'ner_tags': ner_tags}
+
+        return Sample(input=input_text, target=target_text, metadata=metadata)
+
+    def format_prompt_template(self, sample):
+        """
+        Format the prompt with entity types, available tags, and text to annotate.
+        """
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()
+        return self.prompt_template.format(
+            entities=self.entities_description, entity_list=', '.join(self.entity_list), text=sample.input
+        )
+
+    def format_fewshot_template(self, fewshot, sample):
+        """
+        Format the few-shot prompt with all required parameters.
+        """
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()
+        return self.few_shot_prompt_template.format(
+            fewshot=fewshot,
+            entities=self.entities_description,
+            entity_list=', '.join(self.entity_list),
+            text=sample.input
+        )

evalscope/benchmarks/ner/cross_ner_entities/ai.py

@@ -0,0 +1,54 @@
+def get_entity_mappings():
+    entity_type_map = {
+        'ALGORITHM': 'algorithm',
+        'CONFERENCE': 'conference',
+        'COUNTRY': 'country',
+        'FIELD': 'field',
+        'LOCATION': 'location',
+        'METRICS': 'metrics',
+        'MISC': 'misc',
+        'ORGANISATION': 'organisation',
+        'PERSON': 'person',
+        'PRODUCT': 'product',
+        'PROGRAMLANG': 'programming_language',
+        'RESEARCHER': 'researcher',
+        'TASK': 'task',
+        'UNIVERSITY': 'university'
+    }
+    entity_descriptions = {
+        'ALGORITHM':
+        ('A specific algorithm or model architecture in AI (e.g., "Transformer", '
+         '"gradient descent", "ResNet").'),
+        'CONFERENCE': ('An academic conference related to AI (e.g., "NeurIPS", "ICML", "CVPR").'),
+        'COUNTRY': ('A country mentioned in the context of AI research or development '
+                    '(e.g., "USA", "China").'),
+        'FIELD':
+        ('A sub-field or area of study within AI (e.g., "Natural Language Processing", '
+         '"Computer Vision").'),
+        'LOCATION':
+        ('A specific geographical location relevant to AI, other than countries '
+         '(e.g., "Silicon Valley").'),
+        'METRICS': ('A performance metric used to evaluate AI models (e.g., "F1-score", '
+                    '"BLEU", "accuracy").'),
+        'MISC': ('Miscellaneous AI-related terms that don\'t fit other categories '
+                 '(e.g., "Turing Award").'),
+        'ORGANISATION':
+        ('An organization, company, or lab involved in AI (e.g., "Google AI", '
+         '"OpenAI", "DeepMind").'),
+        'PERSON':
+        ('A person mentioned in the context of AI, who is not a researcher '
+         '(e.g., a CEO or public figure).'),
+        'PRODUCT': ('An AI-related product, framework, or software (e.g., "TensorFlow", '
+                    '"PyTorch", "AlphaGo").'),
+        'PROGRAMLANG': ('A programming language used in AI (e.g., "Python", "C++", "Julia").'),
+        'RESEARCHER': ('A person who conducts research in the field of AI (e.g., "Yann LeCun", '
+                       '"Geoffrey Hinton").'),
+        'TASK': (
+            'A specific problem or task that AI is used to solve (e.g., "Image Classification", '
+            '"Sentiment Analysis").'
+        ),
+        'UNIVERSITY':
+        ('A university or academic institution involved in AI research (e.g., '
+         '"Stanford University", "MIT").')
+    }
+    return entity_type_map, entity_descriptions

evalscope/benchmarks/ner/cross_ner_entities/literature.py

@@ -0,0 +1,36 @@
+def get_entity_mappings():
+    entity_type_map = {
+        'AWARD': 'award',
+        'BOOK': 'book',
+        'COUNTRY': 'country',
+        'EVENT': 'event',
+        'LITERARYGENRE': 'literary_genre',
+        'LOCATION': 'location',
+        'MAGAZINE': 'magazine',
+        'MISC': 'misc',
+        'ORGANISATION': 'organisation',
+        'PERSON': 'person',
+        'POEM': 'poem',
+        'WRITER': 'writer'
+    }
+    entity_descriptions = {
+        'AWARD': ('A literary award or prize (e.g., "Nobel Prize in Literature", "Booker Prize").'),
+        'BOOK': ('The title of a book (e.g., "Pride and Prejudice", "One Hundred Years of Solitude").'),
+        'COUNTRY': ('A country relevant to the literary context (e.g., "England", "Russia").'),
+        'EVENT': ('A literary festival or significant event (e.g., "Hay Festival", "Frankfurt Book Fair").'),
+        'LITERARYGENRE':
+        ('A genre or category of literature (e.g., "Science Fiction", "Gothic novel", '
+         '"magical realism").'),
+        'LOCATION': ('A real or fictional place mentioned in a literary context (e.g., "London", '
+                     '"Middle-earth").'),
+        'MAGAZINE': ('A magazine or literary journal (e.g., "The New Yorker", "Paris Review").'),
+        'MISC': ('Miscellaneous literary terms (e.g., "protagonist", "sonnet", '
+                 '"Shakespeare\'s Globe").'),
+        'ORGANISATION': ('A publishing house or literary organization (e.g., "Penguin Random House").'),
+        'PERSON': ('A character or person mentioned who is not a writer (e.g., "Elizabeth Bennet", '
+                   '"King Lear").'),
+        'POEM': ('The title of a poem (e.g., "The Waste Land", "Ozymandias").'),
+        'WRITER': ('The name of a writer, author, or poet (e.g., "Jane Austen", '
+                   '"Gabriel Garcia Marquez").')
+    }
+    return entity_type_map, entity_descriptions

evalscope/benchmarks/ner/cross_ner_entities/music.py

@@ -0,0 +1,39 @@
+def get_entity_mappings():
+    entity_type_map = {
+        'ALBUM': 'album',
+        'AWARD': 'award',
+        'BAND': 'band',
+        'COUNTRY': 'country',
+        'EVENT': 'event',
+        'LOCATION': 'location',
+        'MISC': 'misc',
+        'MUSICALARTIST': 'musical_artist',
+        'MUSICALINSTRUMENT': 'musical_instrument',
+        'MUSICGENRE': 'music_genre',
+        'ORGANISATION': 'organisation',
+        'PERSON': 'person',
+        'SONG': 'song'
+    }
+    entity_descriptions = {
+        'ALBUM': ('The title of a music album (e.g., "Abbey Road", "Thriller", "Lemonade").'),
+        'AWARD': ('A music award or prize (e.g., "Grammy Award", "MTV Music Award").'),
+        'BAND': ('The name of a musical group or band (e.g., "The Beatles", "Queen", "BTS").'),
+        'COUNTRY': ('A country relevant to the music context (e.g., "USA", "UK", "South Korea").'),
+        'EVENT': ('A music festival, concert tour, or event (e.g., "Glastonbury Festival", '
+                  '"Woodstock").'),
+        'LOCATION':
+        ('A venue, studio, or place relevant to music (e.g., "Madison Square Garden", '
+         '"Abbey Road Studios").'),
+        'MISC': ('Miscellaneous music-related terms (e.g., "synthesizer", "major key", '
+                 '"a cappella").'),
+        'MUSICALARTIST': ('A solo musician or singer (e.g., "Michael Jackson", "Taylor Swift", '
+                          '"Ed Sheeran").'),
+        'MUSICALINSTRUMENT': ('A musical instrument (e.g., "guitar", "piano", "violin").'),
+        'MUSICGENRE': ('A genre or style of music (e.g., "Rock", "Pop", "Jazz", "K-Pop").'),
+        'ORGANISATION': ('A record label or music organization (e.g., "Capitol Records", "Sony Music").'),
+        'PERSON':
+        ('A person related to music who is not a primary artist (e.g., a producer, '
+         'a songwriter, "John Lennon").'),
+        'SONG': ('The title of a song (e.g., "Bohemian Rhapsody", "Hey Jude", "Dynamite").')
+    }
+    return entity_type_map, entity_descriptions

evalscope/benchmarks/ner/cross_ner_entities/politics.py

@@ -0,0 +1,37 @@
+def get_entity_mappings():
+    entity_type_map = {
+        'COUNTRY': 'country',
+        'ELECTION': 'election',
+        'EVENT': 'event',
+        'LOCATION': 'location',
+        'MISC': 'misc',
+        'ORGANISATION': 'organisation',
+        'PERSON': 'person',
+        'POLITICALPARTY': 'political_party',
+        'POLITICIAN': 'politician'
+    }
+    entity_descriptions = {
+        'COUNTRY': ('A country or sovereign state (e.g., "United States", "Germany").'),
+        'ELECTION': ('A specific election event (e.g., "2024 presidential election", '
+                     '"midterm elections").'),
+        'EVENT':
+        ('A significant political event, summit, or incident (e.g., "G7 Summit", '
+         '"Brexit", "Watergate scandal").'),
+        'LOCATION':
+        ('A politically significant building or location (e.g., "The White House", '
+         '"10 Downing Street").'),
+        'MISC': (
+            'Miscellaneous political terms, ideologies, or documents (e.g., "democracy", '
+            '"impeachment", "the Constitution").'
+        ),
+        'ORGANISATION':
+        ('A political or governmental organization (e.g., "United Nations", "NATO", '
+         '"European Union").'),
+        'PERSON':
+        ('A person mentioned in a political context who is not a politician '
+         '(e.g., a journalist, an activist).'),
+        'POLITICALPARTY': ('A named political party (e.g., "Democratic Party", "Conservative Party").'),
+        'POLITICIAN': ('A person who holds or seeks political office (e.g., "Joe Biden", '
+                       '"Angela Merkel").')
+    }
+    return entity_type_map, entity_descriptions

evalscope/benchmarks/ner/cross_ner_entities/science.py

@@ -0,0 +1,58 @@
+def get_entity_mappings():
+    entity_type_map = {
+        'ACADEMICJOURNAL': 'academic_journal',
+        'ASTRONOMICALOBJECT': 'astronomical_object',
+        'AWARD': 'award',
+        'CHEMICALCOMPOUND': 'chemical_compound',
+        'CHEMICALELEMENT': 'chemical_element',
+        'COUNTRY': 'country',
+        'DISCIPLINE': 'discipline',
+        'ENZYME': 'enzyme',
+        'EVENT': 'event',
+        'LOCATION': 'location',
+        'MISC': 'misc',
+        'ORGANISATION': 'organisation',
+        'PERSON': 'person',
+        'PROTEIN': 'protein',
+        'SCIENTIST': 'scientist',
+        'THEORY': 'theory',
+        'UNIVERSITY': 'university'
+    }
+    entity_descriptions = {
+        'ACADEMICJOURNAL': ('A scientific journal or publication (e.g., "Nature", "Science", "The Lancet").'),
+        'ASTRONOMICALOBJECT': ('A natural object in space (e.g., "Mars", "Andromeda Galaxy", '
+                               '"Halley\'s Comet").'),
+        'AWARD': ('A scientific award or prize (e.g., "Nobel Prize in Physics", "Fields Medal").'),
+        'CHEMICALCOMPOUND':
+        ('A chemical substance consisting of two or more elements (e.g., "H2O", '
+         '"Carbon Dioxide").'),
+        'CHEMICALELEMENT': ('An element from the periodic table (e.g., "Hydrogen", "Oxygen", "Gold").'),
+        'COUNTRY': ('A country relevant to a scientific context (e.g., "Switzerland" for CERN).'),
+        'DISCIPLINE':
+        ('A branch of science or academic discipline (e.g., "Physics", '
+         '"Molecular Biology", "Astronomy").'),
+        'ENZYME': ('A specific type of protein that acts as a catalyst (e.g., "Lactase", "Catalase").'),
+        'EVENT': ('A significant scientific mission or event (e.g., "Apollo 11 mission", '
+                  '"Human Genome Project").'),
+        'LOCATION':
+        ('A research facility or location of scientific importance (e.g., "CERN", '
+         '"International Space Station").'),
+        'MISC':
+        ('Miscellaneous scientific terms or concepts (e.g., "double helix", '
+         '"black hole", "quantum mechanics").'),
+        'ORGANISATION': ('A scientific organization or agency (e.g., "NASA", "Max Planck Society", "WHO").'),
+        'PERSON':
+        ('A person mentioned in a scientific context who is not a scientist '
+         '(e.g., a patient, a benefactor).'),
+        'PROTEIN': ('A specific protein (that is not an enzyme) (e.g., "Hemoglobin", '
+                    '"Insulin", "Keratin").'),
+        'SCIENTIST':
+        ('A person who is a scientist, researcher, or inventor (e.g., "Albert Einstein", '
+         '"Marie Curie").'),
+        'THEORY': ('A named scientific theory or law (e.g., "Theory of Relativity", '
+                   '"Big Bang Theory").'),
+        'UNIVERSITY':
+        ('A university or academic institution involved in science (e.g., '
+         '"Cambridge University", "Caltech").')
+    }
+    return entity_type_map, entity_descriptions

evalscope/benchmarks/ner/genia_ner_adapter.py

@@ -0,0 +1,66 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'GeniaNER consisting of 2,000 MEDLINE abstracts has been released with more than '
+    '400,000 words and almost 100,000 annotations for biological terms.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='genia_ner',
+        pretty_name='GeniaNER',
+        dataset_id='extraordinarylab/genia-ner',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class GeniaNERAdapter(NERAdapter):
+    """
+    Adapter for the GeniaNER Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the GeniaNER dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define GeniaNER-specific entity mappings
+        self.entity_type_map = {
+            'CELL_LINE': 'cell_line',
+            'CELL_TYPE': 'cell_type',
+            'DNA': 'dna',
+            'PROTEIN': 'protein',
+            'RNA': 'rna'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'CELL_LINE':
+            'A population of cells derived from a single cell and grown in a culture.',
+            'CELL_TYPE':
+            ('A category of cells that are part of a larger organism and share a specific '
+             'structure and function.'),
+            'DNA':
+            'Deoxyribonucleic acid. This includes specific genes, domains, and regions of a DNA molecule.',
+            'PROTEIN': (
+                'Molecules composed of amino acids that perform a vast array of functions within '
+                'organisms. This includes enzymes, receptors, and signaling molecules.'
+            ),
+            'RNA':
+            'Ribonucleic acid. This refers to RNA molecules, including messenger RNA (mRNA) and other types.'
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/harvey_ner_adapter.py

@@ -0,0 +1,58 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'HarveyNER is a dataset with fine-grained locations annotated in tweets. This dataset '
+    'presents unique challenges and characterizes many complex and long location mentions '
+    'in informal descriptions.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='harvey_ner',
+        pretty_name='HarveyNER',
+        dataset_id='extraordinarylab/harvey-ner',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class HarveyNERAdapter(NERAdapter):
+    """
+    Adapter for the HarveyNER Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the HarveyNER dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define HarveyNER-specific entity mappings
+        self.entity_type_map = {'AREA': 'area', 'POINT': 'point', 'RIVER': 'river', 'ROAD': 'road'}
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'AREA':
+            'Geographical entities such as city subdivisions, neighborhoods, etc.',
+            'POINT': (
+                'An exact location that a geocoordinate can be assigned. E.g., a uniquely named '
+                'building, intersections of roads or rivers.'
+            ),
+            'RIVER':
+            'A river or a section of a river.',
+            'ROAD':
+            'A road or a section of a road.'
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/mit_movie_trivia_adapter.py

@@ -0,0 +1,74 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'The MIT-Movie-Trivia dataset, originally created for slot filling, is modified by '
+    'ignoring some slot types (e.g. genre, rating) and merging others (e.g. director '
+    'and actor in person, and song and movie title in title) in order to keep '
+    'consistent named entity types across all datasets.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mit_movie_trivia',
+        pretty_name='MIT-Movie-Trivia',
+        dataset_id='extraordinarylab/mit-movie-trivia',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class MITMovieTriviaAdapter(NERAdapter):
+    """
+    Adapter for the MIT-Movie-Trivia Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the MIT-Movie-Trivia dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define MIT-Movie-Trivia-specific entity mappings
+        self.entity_type_map = {
+            'ACTOR': 'actor',
+            'AWARD': 'award',
+            'CHARACTER_NAME': 'character_name',
+            'DIRECTOR': 'director',
+            'GENRE': 'genre',
+            'OPINION': 'opinion',
+            'ORIGIN': 'origin',
+            'PLOT': 'plot',
+            'QUOTE': 'quote',
+            'RELATIONSHIP': 'relationship',
+            'SOUNDTRACK': 'soundtrack',
+            'YEAR': 'year'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'ACTOR': 'The name of an actor or actress starring in the movie.',
+            'AWARD': 'An award the movie won or was nominated for.',
+            'CHARACTER_NAME': 'The name of a character in the movie.',
+            'DIRECTOR': 'The name of the person who directed the movie.',
+            'GENRE': 'The category or style of the movie.',
+            'OPINION': 'A subjective review or personal opinion about the movie.',
+            'ORIGIN': 'The source material or basis for the movie.',
+            'PLOT': 'A description or summary of the movie\'s storyline.',
+            'QUOTE': 'A memorable line or phrase spoken in the movie.',
+            'RELATIONSHIP': 'The connection or relationship between characters.',
+            'SOUNDTRACK': 'The music or a specific song from the movie.',
+            'YEAR': 'The release year of the movie.'
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/mit_restaurant_adapter.py

@@ -0,0 +1,66 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'The MIT-Restaurant dataset is a collection of restaurant review text specifically '
+    'curated for training and testing Natural Language Processing (NLP) models, '
+    'particularly for Named Entity Recognition (NER). It contains sentences from real '
+    'reviews, along with corresponding labels in the BIO format.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='mit_restaurant',
+        pretty_name='MIT-Restaurant',
+        dataset_id='extraordinarylab/mit-restaurant',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class MITRestaurantAdapter(NERAdapter):
+    """
+    Adapter for the MIT-Restaurant Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the MIT-Restaurant dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define MIT-Restaurant-specific entity mappings
+        self.entity_type_map = {
+            'AMENITY': 'amenity',
+            'CUISINE': 'cuisine',
+            'DISH': 'dish',
+            'HOURS': 'hours',
+            'LOCATION': 'location',
+            'PRICE': 'price',
+            'RATING': 'rating',
+            'RESTAURANT_NAME': 'restaurant_name'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'AMENITY': 'A feature or service offered by the restaurant.',
+            'CUISINE': 'The type of food a restaurant serves.',
+            'DISH': 'A specific food or drink item.',
+            'HOURS': 'The operating hours of a restaurant.',
+            'LOCATION': 'The address or general location of a restaurant.',
+            'PRICE': 'The price range of a restaurant.',
+            'RATING': 'A rating or review of the restaurant.',
+            'RESTAURANT_NAME': 'The name of a restaurant.',
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()