evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/multi_if/metrics.py

@@ -0,0 +1,120 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from typing import Any, Dict, List, Optional, Tuple
+
+from evalscope.utils import get_logger
+from . import ifeval
+
+logger = get_logger()
+
+
+def gen_acc_strict(x: Dict[str, Any]) -> Dict[str, List]:
+    # reference: fbcode/gen_ai/github/fair_evals/evals/tasks/finetune/ifeval.py
+    response = str(x['response'])
+    instruction_list = x['instruction_id_list']
+    is_following_list = []
+    for index, instruction_id in enumerate(instruction_list):
+        instruction_cls = ifeval.INSTRUCTION_DICT[instruction_id]
+        instruction = instruction_cls(instruction_id)
+
+        instruction.build_description(**x['kwargs'][index])
+        if response and instruction.check_following(response):
+            is_following_list.append(True)
+        else:
+            is_following_list.append(False)
+
+    return {
+        'follow_instruction_list': is_following_list,
+        'instruction_id_list': instruction_list,
+    }
+
+
+def gen_acc_loose(x: Dict[str, Any]) -> Dict[str, List]:
+    response = str(x['response'])
+    r = response.split('\n')
+    response_remove_first = '\n'.join(r[1:]).strip()
+    response_remove_last = '\n'.join(r[:-1]).strip()
+    response_remove_both = '\n'.join(r[1:-1]).strip()
+    revised_response = response.replace('*', '')
+    revised_response_remove_first = response_remove_first.replace('*', '')
+    revised_response_remove_last = response_remove_last.replace('*', '')
+    revised_response_remove_both = response_remove_both.replace('*', '')
+    all_responses = [
+        response,
+        revised_response,
+        response_remove_first,
+        response_remove_last,
+        response_remove_both,
+        revised_response_remove_first,
+        revised_response_remove_last,
+        revised_response_remove_both,
+    ]
+    instruction_list = x['instruction_id_list']
+    is_following_list = []
+    for index, instruction_id in enumerate(instruction_list):
+        instruction_cls = ifeval.INSTRUCTION_DICT[instruction_id]
+        instruction = instruction_cls(instruction_id)
+
+        instruction.build_description(**x['kwargs'][index])
+
+        is_following = False
+        for r in all_responses:  # type: ignore
+            if r.strip() and instruction.check_following(r):  # type: ignore
+                is_following = True
+                break
+
+        is_following_list.append(is_following)
+    return {
+        'follow_instruction_list': is_following_list,
+        'instruction_id_list': instruction_list,
+    }
+
+
+def parse_result(outputs: List[Dict[str, Any]]) -> Tuple[float, float]:
+
+    prompt_total = 0
+    prompt_correct = 0
+    instruction_total = 0
+    instruction_correct = 0
+
+    for example in outputs:
+        follow_instruction_list = example['follow_instruction_list']
+        instruction_id_list = example['instruction_id_list']
+
+        prompt_total += 1
+        if all(follow_instruction_list):
+            prompt_correct += 1
+
+        instruction_total += len(instruction_id_list)
+        instruction_correct += sum(follow_instruction_list)
+
+    return prompt_correct / prompt_total if prompt_total > 0 else 0, \
+        instruction_correct / instruction_total if instruction_total > 0 else 0
+
+
+def parse_result_no_reduce(outputs: List[Dict[str, Any]]) -> Tuple[List, List]:
+
+    prompt_res = []
+    inst_res = []
+
+    for example in outputs:
+        follow_instruction_list = example['follow_instruction_list']
+        instruction_id_list = example['instruction_id_list']
+        if all(follow_instruction_list):
+            prompt_res.append(1)
+        else:
+            prompt_res.append(0)
+        inst_res.append(sum(follow_instruction_list) / len(instruction_id_list) if instruction_id_list else 0.0)
+
+    return prompt_res, inst_res

evalscope/benchmarks/multi_if/multi_if_adapter.py

@@ -0,0 +1,161 @@
+import json
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, messages_pretty_str
+from evalscope.api.metric import Score
+from evalscope.api.model import Model
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.import_utils import check_import
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'Chinese',
+    'English',
+    'German',
+    'Italian',
+    'Vietnamese',
+    'Spanish',
+    'Hindi',
+    'Portuguese',
+    'French',
+    'Thai',
+    'Russian',
+]
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='multi_if',
+        pretty_name='Multi-IF',
+        description=
+        'Multi-IF is a benchmark designed to evaluate the performance of LLM models\' capabilities in multi-turn instruction following within a multilingual environment.',  # noqa: E501
+        tags=[Tags.INSTRUCTION_FOLLOWING, Tags.MULTI_LINGUAL, Tags.MULTI_TURN],
+        dataset_id='facebook/Multi-IF',
+        subset_list=SUBSET_LIST,
+        metric_list=[
+            'prompt_level_strict',
+            'inst_level_strict',
+            'prompt_level_loose',
+            'inst_level_loose',
+        ],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',
+        extra_params={
+            'max_turns': 3,  # maximum number of turns to evaluate
+        }
+    )
+)
+class MultiIFAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        # Ensure required packages are installed
+        check_import(
+            module_name=['nltk', 'langdetect'],
+            package=['nltk', 'langdetect'],
+            raise_error=True,
+            feature_name=self.pretty_name
+        )
+        if 'Chinese' in self.subset_list:
+            check_import(module_name='emoji', package='emoji', raise_error=True, feature_name='Chinese subset')
+        if 'Thai' in self.subset_list:
+            check_import(module_name='pythainlp', package='pythainlp', raise_error=True, feature_name='Thai subset')
+
+        self.reformat_subset = True
+        self.max_turns = self.extra_params.get('max_turns', 3)
+        if not isinstance(self.max_turns, int) or self.max_turns < 1 or self.max_turns > 3:
+            logger.warning(f'max_turns should be an integer between 1 and 3, got {self.max_turns}, clamping to 3.')
+            self.max_turns = 3
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=[ChatMessageUser(content='')],  # NOTE: we will build the multi turn conversation in the evaluator
+            target='',
+            subset_key=record['language'],
+            metadata=record,
+        )
+
+    def run_inference(self, model: Model, sample: Sample, output_dir: str, **kwargs) -> TaskState:
+        """
+        Run multi-turn inference with the model and sample.
+        """
+        record = sample.metadata
+        history = []
+        step_record = {}
+        for step in range(1, self.max_turns + 1):
+            current_prompt = json.loads(record[f'turn_{step}_prompt'])
+            history.append(ChatMessageUser(content=current_prompt['content']))
+            # Generate model output
+            model_output = model.generate(input=history, tools=sample.tools)
+
+            response = model_output.completion
+            instruction_id_list = json.loads(record[f'turn_{step}_instruction_id_list'])
+            kwargs_list = json.loads(record[f'turn_{step}_kwargs'])
+            _kwargs = [json.loads(kwarg) for kwarg in kwargs_list]
+
+            step_record[step] = {
+                'prompt': messages_pretty_str(history),
+                'response': response,
+                'instruction_id_list': instruction_id_list,
+                'kwargs': _kwargs
+            }
+
+            # Append model output to history for next turn
+            history.append(model_output.message)
+
+        sample.metadata['step_record'] = step_record
+        return TaskState(
+            model=model.name,
+            sample=sample,
+            messages=history,
+            output=model_output,
+            completed=True,
+        )
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: Dict, task_state: TaskState
+    ) -> Score:
+        """
+        Calculate evaluation scores by comparing prediction with reference.
+        """
+        from .metrics import gen_acc_loose, gen_acc_strict, parse_result
+
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        step_record = task_state.metadata['step_record']
+        results = {}
+        try:
+            for step, record in step_record.items():
+                outputs_strict = gen_acc_strict(record)
+                outputs_loose = gen_acc_loose(record)
+                prompt_level_strict, inst_level_strict = parse_result([outputs_strict])
+                prompt_level_loose, inst_level_loose = parse_result([outputs_loose])
+                results.update({
+                    f'turn_{step}_prompt_level_strict': prompt_level_strict,
+                    f'turn_{step}_inst_level_strict': inst_level_strict,
+                    f'turn_{step}_prompt_level_loose': prompt_level_loose,
+                    f'turn_{step}_inst_level_loose': inst_level_loose,
+                })
+            score.value.update(results)
+
+            # Set main score name
+            if results:
+                score.main_score_name = f'turn_{step}_prompt_level_strict'
+
+        except Exception as e:
+            logger.error(f'Error calculating ifeval metrics: {e}')
+            score.value = {}
+
+        return score

evalscope/benchmarks/music_trivia/music_trivia_adapter.py

@@ -0,0 +1,36 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+DESCRIPTION = (
+    'MusicTrivia is a curated dataset of multiple-choice questions covering both classical and modern music topics. '
+    'It includes questions about composers, musical periods, and popular artists, designed for evaluating '
+    'factual recall and domain-specific music knowledge.'
+)  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='music_trivia',
+        pretty_name='MusicTrivia',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/music-trivia',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
+)
+class MusicTriviaAdapter(MultiChoiceAdapter):
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )

evalscope/benchmarks/needle_haystack/needle_haystack_adapter.py

@@ -36,7 +36,7 @@ Don't give information outside the document or repeat your findings."""
         tags=[Tags.RETRIEVAL, Tags.LONG_CONTEXT],
         description='Needle in a Haystack is a benchmark focused on information retrieval tasks. '
         'It requires the model to find specific information within a large corpus of text. '
-        '[Usage Example](https://evalscope.readthedocs.io/zh-cn/latest/third_party/needle_haystack.html)',  # noqa: E501
+        '[Usage Example](https://evalscope.readthedocs.io/en/latest/third_party/needle_haystack.html)',  # noqa: E501
         dataset_id='AI-ModelScope/Needle-in-a-Haystack-Corpus',
         metric_list=['acc'],
         subset_list=['english', 'chinese'],
@@ -73,6 +73,7 @@ class NeedleHaystackAdapter(DefaultDataAdapter):
         super().__init__(**kwargs)
 
         self._use_llm_judge = True
+        self.add_aggregation_name = False  # Don't add aggregation name for needle haystack adapter
         # set extra params
         self.retrieval_question = self.extra_params.get(
             'retrieval_question', 'What is the best thing to do in San Francisco?'
@@ -164,7 +165,11 @@ class NeedleHaystackAdapter(DefaultDataAdapter):
                     records.append(record)
 
                 dataset = DictDataLoader(
-                    dict_list=records, limit=self.limit, repeats=self.repeats, sample_fields=self.record_to_sample
+                    dict_list=records,
+                    limit=self.limit,
+                    repeats=self.repeats,
+                    sample_fields=self.record_to_sample,
+                    shuffle=self.shuffle,
                 ).load()
 
                 datasets[subset_name] = dataset
@@ -355,10 +360,6 @@ class NeedleHaystackAdapter(DefaultDataAdapter):
 
         return score
 
-    def _on_generate_report(self, scores, model_name, add_aggregation_name=True):
-        # Don't add aggregation name for needle haystack adapter
-        return super()._on_generate_report(scores, model_name, False)
-
     def _on_generate_report_end(self, report: 'Report', output_dir: str, **kwargs):
         try:
             import os

evalscope/benchmarks/ner/broad_twitter_corpus_adapter.py

@@ -0,0 +1,52 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'BroadTwitterCorpus is a dataset of tweets collected over stratified times, places '
+    'and social uses. The goal is to represent a broad range of activities, giving a '
+    'dataset more representative of the language used in this hardest of social media '
+    'formats to process.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='broad_twitter_corpus',
+        pretty_name='BroadTwitterCorpus',
+        dataset_id='extraordinarylab/broad-twitter-corpus',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class BroadTwitterCorpusAdapter(NERAdapter):
+    """
+    Adapter for the BroadTwitterCorpus Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the BroadTwitterCorpus dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define BroadTwitterCorpus-specific entity mappings
+        self.entity_type_map = {'PER': 'person', 'ORG': 'organization', 'LOC': 'location'}
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'PER': 'Names of people, including first and last names',
+            'ORG': 'Names of companies, institutions, organizations, etc.',
+            'LOC': 'Names of locations, cities, states, countries, etc.',
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/conll2003_adapter.py

@@ -0,0 +1,48 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='conll2003',
+        pretty_name='CoNLL2003',
+        dataset_id='evalscope/conll2003',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description='The ConLL-2003 dataset is for the Named Entity Recognition (NER) task. It was introduced as part '
+        'of the ConLL-2003 Shared Task conference and contains texts annotated with entities such as '
+        'people, organizations, places, and various names.',
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class CoNLL2003Adapter(NERAdapter):
+    """
+    Adapter for the CoNLL2003 Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the CoNLL2003 dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define CoNLL2003-specific entity mappings
+        self.entity_type_map = {'PER': 'person', 'ORG': 'organization', 'LOC': 'location', 'MISC': 'miscellaneous'}
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'PER': 'Names of people, including first and last names',
+            'ORG': 'Names of companies, institutions, organizations, etc.',
+            'LOC': 'Names of locations, cities, states, countries, etc.',
+            'MISC': 'Miscellaneous entities not in the above categories'
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/copious_adapter.py

@@ -0,0 +1,85 @@
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE
+
+DESCRIPTION = (
+    'Copious corpus is a gold standard corpus that covers a wide range of biodiversity '
+    'entities, consisting of 668 documents downloaded from the Biodiversity Heritage '
+    'Library with over 26K sentences and more than 28K entities.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='copious',
+        pretty_name='Copious',
+        dataset_id='extraordinarylab/copious',
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class CopiousAdapter(NERAdapter):
+    """
+    Adapter for the Copious Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the Copious dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define Copious-specific entity mappings
+        self.entity_type_map = {
+            'TAXON': 'taxon',
+            'GEOGRAPHICAL_LOCATION': 'geographical_location',
+            'HABITAT': 'habitat',
+            'PERSON': 'person',
+            'TEMPORAL_EXPRESSION': 'temporal_expression'
+        }
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {
+            'TAXON': (
+                'Mentions of taxonomic ranks such as species, genus, and family. '
+                'This includes scientific names (e.g., "Salvelinus alpinus") and '
+                'vernacular names (e.g., "flying fox"), but excludes general terms '
+                'like "fish" or "birds" and microorganism names.'
+            ),
+            'GEOGRAPHICAL_LOCATION': (
+                'Identifiable points or areas on the planet, including continents, '
+                'countries, cities, landforms, and bodies of water (e.g., "East coast '
+                'of Mindoro", "Balayan Bay"). This also includes geographical '
+                'coordinates (e.g., "13o 36\' 11\\" N.").'
+            ),
+            'HABITAT': (
+                'Descriptions of environments where organisms live. This includes '
+                'natural environments (e.g., "Lowland forest", "subalpine calcareous '
+                'pastures") and places where parasites or epiphytes reside (e.g., '
+                '"parasitic on Achillea holosericea"). It excludes habitat attributes '
+                'like altitude or depth.'
+            ),
+            'PERSON': (
+                'Proper nouns referring to person names, including those in historical '
+                'accounts or citations related to a species observation (e.g., "In 1905, '
+                '[Tattersall] follows..."). It excludes titles, general references like '
+                '"the researcher", and names that are part of a taxon\'s authority.'
+            ),
+            'TEMPORAL_EXPRESSION': (
+                'Spans of text referring to points in time. This includes specific dates '
+                '(e.g., "10 June 2013"), years, decades, seasons, and geochronological ages '
+                '(e.g., "late Pleistocene"). It excludes time-of-day information and dates '
+                'within a taxon name\'s authority.'
+            )
+        }
+
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()

evalscope/benchmarks/ner/cross_ner_adapter.py

@@ -0,0 +1,120 @@
+from typing import Any, Dict, List, Set, Tuple
+
+from evalscope.api.benchmark import BenchmarkMeta, NERAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.benchmarks.ner.cross_ner_entities import ai, literature, music, politics, science
+from evalscope.constants import Tags
+from evalscope.utils.ner import FEWSHOT_TEMPLATE, PROMPT_TEMPLATE, create_target_text
+
+DESCRIPTION = (
+    'CrossNER is a fully-labelled collected of named entity recognition (NER) data '
+    'spanning over five diverse domains (AI, Literature, Music, Politics, Science).'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='cross_ner',
+        pretty_name='CrossNER',
+        dataset_id='extraordinarylab/cross-ner',
+        subset_list=['ai', 'literature', 'music', 'politics', 'science'],
+        tags=[Tags.KNOWLEDGE, Tags.NER],
+        description=DESCRIPTION.strip(),
+        few_shot_num=5,
+        train_split='train',
+        eval_split='test',
+        metric_list=['precision', 'recall', 'f1_score', 'accuracy'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class CrossNERAdapter(NERAdapter):
+    """
+    Adapter for the CrossNER Named Entity Recognition dataset.
+
+    This adapter inherits the NER functionality from NERAdapter and
+    configures it specifically for the CrossNER dataset's entity types.
+    """
+
+    def __init__(self, **kwargs):
+        # Initialize the parent class first
+        super().__init__(**kwargs)
+
+        # Define CrossNER-specific entity mappings
+        self.entity_type_map = {}
+
+        # Add descriptions for each entity type
+        self.entity_descriptions = {}
+
+    def setup_entity_mappings(self):
+        """
+        Setup entity mappings and descriptions for prompt formatting.
+        This should be called after entity_type_map and entity_descriptions are defined.
+        """
+        if self.current_subset_name == 'ai':
+            self.entity_type_map, self.entity_descriptions = ai.get_entity_mappings()
+        elif self.current_subset_name == 'literature':
+            self.entity_type_map, self.entity_descriptions = literature.get_entity_mappings()
+        elif self.current_subset_name == 'music':
+            self.entity_type_map, self.entity_descriptions = music.get_entity_mappings()
+        elif self.current_subset_name == 'politics':
+            self.entity_type_map, self.entity_descriptions = politics.get_entity_mappings()
+        elif self.current_subset_name == 'science':
+            self.entity_type_map, self.entity_descriptions = science.get_entity_mappings()
+
+        # Reverse mapping for converting back from prediction to evaluation
+        self.reverse_entity_map = {v.lower(): k for k, v in self.entity_type_map.items()}
+
+        # Create list of tags for prompt formatting
+        self.entity_list = [f'<{ent.lower()}>' for ent in self.entity_type_map.values()]
+
+        # Create description of entities for prompt
+        self.entities_description = ', '.join([
+            f'{self.entity_type_map[tag]} ({self.entity_descriptions[tag]})' for tag in self.entity_type_map
+        ])
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a record with tokens and NER tags into a Sample.
+        Creates both the raw text input and annotated text target.
+        """
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()
+
+        tokens: List[str] = record['tokens']
+        ner_tags: List[str] = record['ner_tags']
+
+        # Create the input text by joining tokens
+        input_text = ' '.join(tokens)
+
+        # Process tokens and tags to create annotated target text
+        target_text = create_target_text(tokens, ner_tags, self.entity_type_map)
+
+        # Store tokens and tags in metadata for evaluation
+        metadata = {'tokens': tokens, 'ner_tags': ner_tags}
+
+        return Sample(input=input_text, target=target_text, metadata=metadata)
+
+    def format_prompt_template(self, sample):
+        """
+        Format the prompt with entity types, available tags, and text to annotate.
+        """
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()
+        return self.prompt_template.format(
+            entities=self.entities_description, entity_list=', '.join(self.entity_list), text=sample.input
+        )
+
+    def format_fewshot_template(self, fewshot, sample):
+        """
+        Format the few-shot prompt with all required parameters.
+        """
+        # Setup entity mappings based on the defined entity types
+        self.setup_entity_mappings()
+        return self.few_shot_prompt_template.format(
+            fewshot=fewshot,
+            entities=self.entities_description,
+            entity_list=', '.join(self.entity_list),
+            text=sample.input
+        )