evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/ifeval/ifeval_adapter.py

@@ -1,57 +1,83 @@
-from collections import defaultdict
 from typing import Any, Dict, List
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.benchmarks.ifeval.utils import agg_inst_level_acc, process_results
-from evalscope.constants import EvalType
-from evalscope.metrics import Metric, mean
-from evalscope.models import ChatGenerationModelAdapter
-from evalscope.utils.utils import normalize_score
-
-
-@Benchmark.register(
-    name='ifeval',
-    dataset_id='opencompass/ifeval',
-    model_adapter=ChatGenerationModelAdapter,
-    subset_list=['default'],
-    metric_list=[
-        Metric(name='prompt_level_strict_acc', object=mean),
-        Metric(name='inst_level_strict_acc', object=agg_inst_level_acc),
-        Metric(name='prompt_level_loose_acc', object=mean),
-        Metric(name='inst_level_loose_acc', object=agg_inst_level_acc),
-    ],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='train',
-    prompt_template='',
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ifeval',
+        pretty_name='IFEval',
+        description=
+        'IFEval is a benchmark for evaluating instruction-following language models, focusing on their ability to understand and respond to various prompts. It includes a diverse set of tasks and metrics to assess model performance comprehensively.',  # noqa: E501
+        tags=[Tags.INSTRUCTION_FOLLOWING],
+        dataset_id='opencompass/ifeval',
+        subset_list=['default'],
+        metric_list=[
+            'prompt_level_strict',
+            'inst_level_strict',
+            'prompt_level_loose',
+            'inst_level_loose',
+        ],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',
+        prompt_template='',
+    )
 )
-class IFEvalAdapter(DataAdapter):
+class IFEvalAdapter(DefaultDataAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
-        return {'data': [input_d['prompt']], 'system_prompt': self.prompt_template}
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        prompt = record.get('prompt', '')
+        message_list = [ChatMessageUser(content=prompt)]
+
+        return Sample(input=message_list, target='', metadata=record)
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: Dict, task_state: TaskState
+    ) -> Score:
+        """
+        Calculate evaluation scores by comparing prediction with reference.
+        """
+        from evalscope.benchmarks.ifeval.utils import process_results
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        return result
+        doc = task_state.metadata
+        try:
+            # Process results using the existing ifeval utility
+            results = process_results(doc, [filtered_prediction])
+            score.value.update(results)
 
-    def match(self, gold: Any, pred: Any) -> Dict:
-        return process_results(gold, [pred])
+            # Set main score name
+            score.main_score_name = 'prompt_level_strict'
 
-    def compute_metric(self, review_res_list: List[dict]) -> Any:
-        # aggregate review results
-        res_dict = defaultdict(list)
-        for res in review_res_list:
-            for k, v in res.items():
-                res_dict[k].append(v)
+        except Exception as e:
+            logger.error(f'Error calculating ifeval metrics: {e}')
+            score.value = {}
 
-        metrics = []
-        for metric in self.metric_list:
-            metric_name = metric.name
-            pred_value = res_dict[metric_name]
-            metrics.append({'metric_name': metric_name, 'score': metric.object(pred_value), 'num': len(pred_value)})
-        return metrics
+        return score

evalscope/benchmarks/ifeval/instructions.py

@@ -15,14 +15,13 @@
 
 import collections
 import json
-import langdetect
 import logging
 import random
 import re
 import string
 from typing import Dict, Optional, Sequence, Union
 
-from evalscope.benchmarks.ifeval import instructions_util
+from . import instructions_util
 
 _InstructionArgsDtype = Optional[Dict[str, Union[int, str, Sequence[str]]]]
 
@@ -141,8 +140,9 @@ class ResponseLanguageChecker(Instruction):
         if self._language is None:
             self._language = random.choice(list(_LANGUAGES.keys()))
         # TODO(tianjianlu): opens the description generation to more choices.
-        self._description_pattern = ('Your ENTIRE response should be in {language} language, no other '
-                                     + 'language is allowed.')
+        self._description_pattern = (
+            'Your ENTIRE response should be in {language} language, no other ' + 'language is allowed.'
+        )
         return self._description_pattern.format(language=_LANGUAGES[self._language])
 
     def get_instruction_args(self):
@@ -163,7 +163,7 @@ class ResponseLanguageChecker(Instruction):
           True if the language of `value` follows instruction; otherwise False.
         """
         assert isinstance(value, str)
-
+        import langdetect
         try:
             return langdetect.detect(value) == self._language
         except langdetect.LangDetectException as e:
@@ -198,8 +198,10 @@ class NumberOfSentences(Instruction):
         if relation is None:
             self._comparison_relation = random.choice(_COMPARISON_RELATION)
         elif relation not in _COMPARISON_RELATION:
-            raise ValueError('The supported relation for comparison must be in '
-                             f'{_COMPARISON_RELATION}, but {relation} is given.')
+            raise ValueError(
+                'The supported relation for comparison must be in '
+                f'{_COMPARISON_RELATION}, but {relation} is given.'
+            )
         else:
             self._comparison_relation = relation
 
@@ -256,8 +258,10 @@ class PlaceholderChecker(Instruction):
         self._num_placeholders = num_placeholders
         if self._num_placeholders is None or self._num_placeholders < 0:
             self._num_placeholders = random.randint(1, _NUM_PLACEHOLDERS)
-        self._description_pattern = ('The response must contain at least {num_placeholders} placeholders '
-                                     + 'represented by square brackets, such as [address].')
+        self._description_pattern = (
+            'The response must contain at least {num_placeholders} placeholders '
+            + 'represented by square brackets, such as [address].'
+        )
         return self._description_pattern.format(num_placeholders=self._num_placeholders)
 
     def get_instruction_args(self):
@@ -299,9 +303,10 @@ class BulletListChecker(Instruction):
         self._num_bullets = num_bullets
         if self._num_bullets is None or self._num_bullets < 0:
             self._num_bullets = random.randint(1, _NUM_BULLETS)
-        self._description_pattern = ('Your answer must contain exactly {num_bullets} bullet points. '
-                                     + 'Use the markdown bullet points such as:\n' + '* This is point 1. \n'
-                                     + '* This is point 2')
+        self._description_pattern = (
+            'Your answer must contain exactly {num_bullets} bullet points. '
+            + 'Use the markdown bullet points such as:\n' + '* This is point 1. \n' + '* This is point 2'
+        )
         return self._description_pattern.format(num_bullets=self._num_bullets)
 
     def get_instruction_args(self):
@@ -380,8 +385,9 @@ class ConstrainedStartChecker(Instruction):
         self._starter = starter.strip() if isinstance(starter, str) else starter
         if self._starter is None:
             self._starter = random.choice(_STARTER_OPTIONS)
-        self._description_pattern = ('During the conversation, when it is your turn, '
-                                     + 'please always start with {starter}')
+        self._description_pattern = (
+            'During the conversation, when it is your turn, ' + 'please always start with {starter}'
+        )
         return self._description_pattern.format(starter=self._starter)
 
     def get_instruction_args(self):
@@ -424,8 +430,10 @@ class HighlightSectionChecker(Instruction):
         if self._num_highlights is None or self._num_highlights < 0:
             self._num_highlights = random.randint(1, _NUM_HIGHLIGHTED_SECTIONS)
 
-        self._description_pattern = ('Highlight at least {num_highlights} sections in your answer with '
-                                     + 'markdown, i.e. *highlighted section*.')
+        self._description_pattern = (
+            'Highlight at least {num_highlights} sections in your answer with '
+            + 'markdown, i.e. *highlighted section*.'
+        )
 
         return self._description_pattern.format(num_highlights=self._num_highlights)
 
@@ -483,9 +491,11 @@ class SectionChecker(Instruction):
         if self._num_sections is None or self._num_sections < 0:
             self._num_sections = random.randint(1, _NUM_SECTIONS)
 
-        self._description_pattern = ('Your response must have {num_sections} sections. Mark the beginning '
-                                     + 'of each section with {section_spliter} X, such as:\n' + '{section_spliter} 1\n'
-                                     + '[content of section 1]\n' + '{section_spliter} 2\n' + '[content of section 2]')
+        self._description_pattern = (
+            'Your response must have {num_sections} sections. Mark the beginning '
+            + 'of each section with {section_spliter} X, such as:\n' + '{section_spliter} 1\n'
+            + '[content of section 1]\n' + '{section_spliter} 2\n' + '[content of section 2]'
+        )
 
         return self._description_pattern.format(num_sections=self._num_sections, section_spliter=self._section_spliter)
 
@@ -535,8 +545,9 @@ class ParagraphChecker(Instruction):
         if self._num_paragraphs is None or self._num_paragraphs < 0:
             self._num_paragraphs = random.randint(1, _NUM_PARAGRAPHS)
 
-        self._description_pattern = ('There should be {num_paragraphs} paragraphs. '
-                                     + 'Paragraphs are separated with the markdown divider: ***')
+        self._description_pattern = (
+            'There should be {num_paragraphs} paragraphs. ' + 'Paragraphs are separated with the markdown divider: ***'
+        )
 
         return self._description_pattern.format(num_paragraphs=self._num_paragraphs)
 
@@ -586,12 +597,14 @@ class PostscriptChecker(Instruction):
           A string representing the instruction description.
         """
         self._postscript_marker = (
-            postscript_marker.strip() if isinstance(postscript_marker, str) else postscript_marker)
+            postscript_marker.strip() if isinstance(postscript_marker, str) else postscript_marker
+        )
         if self._postscript_marker is None:
             self._postscript_marker = random.choice(_POSTSCRIPT_MARKER)
 
-        self._description_pattern = ('At the end of your response, please explicitly add a postscript '
-                                     + 'starting with {postscript}')
+        self._description_pattern = (
+            'At the end of your response, please explicitly add a postscript ' + 'starting with {postscript}'
+        )
 
         return self._description_pattern.format(postscript=self._postscript_marker)
 
@@ -645,8 +658,10 @@ class RephraseChecker(Instruction):
                              'in the form of *change me*.')
 
         self._reference_without_change = original_message
-        self._description = ('Rephrasing: Your rephrased response should only'
-                             + 'change the words/sentences in between two asterisks' + 'such as *change me*.')
+        self._description = (
+            'Rephrasing: Your rephrased response should only' + 'change the words/sentences in between two asterisks'
+            + 'such as *change me*.'
+        )
         return self._description
 
     def get_instruction_args(self):
@@ -758,13 +773,16 @@ class KeywordFrequencyChecker(Instruction):
         if relation is None:
             self._comparison_relation = random.choice(_COMPARISON_RELATION)
         elif relation not in _COMPARISON_RELATION:
-            raise ValueError('The supported relation for comparison must be in '
-                             f'{_COMPARISON_RELATION}, but {relation} is given.')
+            raise ValueError(
+                'The supported relation for comparison must be in '
+                f'{_COMPARISON_RELATION}, but {relation} is given.'
+            )
         else:
             self._comparison_relation = relation
 
-        self._description_pattern = ('In your response, the word {keyword} should appear {relation} '
-                                     + '{frequency} times.')
+        self._description_pattern = (
+            'In your response, the word {keyword} should appear {relation} ' + '{frequency} times.'
+        )
 
         return self._description_pattern.format(
             keyword=self._keyword,
@@ -820,8 +838,10 @@ class NumberOfWords(Instruction):
         if relation is None:
             self._comparison_relation = random.choice(_COMPARISON_RELATION)
         elif relation not in _COMPARISON_RELATION:
-            raise ValueError('The supported relation for comparison must be in '
-                             f'{_COMPARISON_RELATION}, but {relation} is given.')
+            raise ValueError(
+                'The supported relation for comparison must be in '
+                f'{_COMPARISON_RELATION}, but {relation} is given.'
+            )
         else:
             self._comparison_relation = relation
 
@@ -851,8 +871,10 @@ class JsonFormat(Instruction):
     """Check the Json format."""
 
     def build_description(self):
-        self._description_pattern = ('Entire output should be wrapped in JSON format. You can use markdown'
-                                     ' ticks such as ```.')
+        self._description_pattern = (
+            'Entire output should be wrapped in JSON format. You can use markdown'
+            ' ticks such as ```.'
+        )
         return self._description_pattern
 
     def get_instruction_args(self):
@@ -865,8 +887,9 @@ class JsonFormat(Instruction):
 
     def check_following(self, value):
         value = (
-            value.strip().removeprefix('```json').removeprefix('```Json').removeprefix('```JSON').removeprefix(
-                '```').removesuffix('```').strip())
+            value.strip().removeprefix('```json').removeprefix('```Json').removeprefix('```JSON').removeprefix('```').
+            removesuffix('```').strip()
+        )
         try:
             json.loads(value)
         except ValueError:
@@ -904,10 +927,12 @@ class ParagraphFirstWordCheck(Instruction):
             self._first_word = instructions_util.generate_keywords(num_keywords=1)[0]
         self._first_word = self._first_word.lower()
 
-        self._description_pattern = ('There should be {num_paragraphs} paragraphs. '
-                                     + 'Paragraphs and only paragraphs are separated with each other by two '
-                                     + "new lines as if it was '\\n\\n' in python. "
-                                     + 'Paragraph {nth_paragraph} must start with word {first_word}.')
+        self._description_pattern = (
+            'There should be {num_paragraphs} paragraphs. '
+            + 'Paragraphs and only paragraphs are separated with each other by two '
+            + "new lines as if it was '\\n\\n' in python. "
+            + 'Paragraph {nth_paragraph} must start with word {first_word}.'
+        )
 
         return self._description_pattern.format(
             num_paragraphs=self._num_paragraphs,
@@ -1085,11 +1110,12 @@ class RephraseParagraph(Instruction):
         self._low = low
         self._high = high
 
-        self._description = ('Rephrase the following paragraph: ' + '{original_paragraph}\nYour response should have '
-                             + 'between {low} and {high} of the same words. '
-                             + 'Words are the same if and only if all of the '
-                             + 'letters, ignoring cases, are the same. For '
-                             + "example, 'run' is the same as 'Run' but different " + "to 'ran'.")
+        self._description = (
+            'Rephrase the following paragraph: ' + '{original_paragraph}\nYour response should have '
+            + 'between {low} and {high} of the same words. ' + 'Words are the same if and only if all of the '
+            + 'letters, ignoring cases, are the same. For ' + "example, 'run' is the same as 'Run' but different "
+            + "to 'ran'."
+        )
 
         return self._description.format(original_paragraph=original_paragraph, low=self._low, high=self._high)
 
@@ -1124,8 +1150,10 @@ class TwoResponsesChecker(Instruction):
 
     def build_description(self):
         """Build the instruction description."""
-        self._description_pattern = ('Give two different responses. Responses and only responses should'
-                                     ' be separated by 6 asterisk symbols: ******.')
+        self._description_pattern = (
+            'Give two different responses. Responses and only responses should'
+            ' be separated by 6 asterisk symbols: ******.'
+        )
         return self._description_pattern
 
     def get_instruction_args(self):
@@ -1172,10 +1200,12 @@ class RepeatPromptThenAnswer(Instruction):
             raise ValueError('prompt_to_repeat must be set.')
         else:
             self._prompt_to_repeat = prompt_to_repeat
-        self._description_pattern = ('First repeat the request word for word without change,'
-                                     ' then give your answer (1. do not say any words or characters'
-                                     ' before repeating the request; 2. the request you need to repeat'
-                                     ' does not include this sentence)')
+        self._description_pattern = (
+            'First repeat the request word for word without change,'
+            ' then give your answer (1. do not say any words or characters'
+            ' before repeating the request; 2. the request you need to repeat'
+            ' does not include this sentence)'
+        )
         return self._description_pattern
 
     def get_instruction_args(self):
@@ -1206,8 +1236,10 @@ class EndChecker(Instruction):
         self._end_phrase = (end_phrase.strip() if isinstance(end_phrase, str) else end_phrase)
         if self._end_phrase is None:
             self._end_phrase = random.choice(_ENDING_OPTIONS)
-        self._description_pattern = ('Finish your response with this exact phrase {ender}. '
-                                     'No other words should follow this phrase.')
+        self._description_pattern = (
+            'Finish your response with this exact phrase {ender}. '
+            'No other words should follow this phrase.'
+        )
         return self._description_pattern.format(ender=self._end_phrase)
 
     def get_instruction_args(self):
@@ -1229,8 +1261,10 @@ class TitleChecker(Instruction):
 
     def build_description(self):
         """Build the instruction description."""
-        self._description_pattern = ('Your answer must contain a title, wrapped in double angular brackets,'
-                                     ' such as <<poem of joy>>.')
+        self._description_pattern = (
+            'Your answer must contain a title, wrapped in double angular brackets,'
+            ' such as <<poem of joy>>.'
+        )
         return self._description_pattern
 
     def get_instruction_args(self):
@@ -1284,13 +1318,17 @@ class LetterFrequencyChecker(Instruction):
         if let_relation is None:
             self._comparison_relation = random.choice(_COMPARISON_RELATION)
         elif let_relation not in _COMPARISON_RELATION:
-            raise ValueError('The supported relation for comparison must be in '
-                             f'{_COMPARISON_RELATION}, but {let_relation} is given.')
+            raise ValueError(
+                'The supported relation for comparison must be in '
+                f'{_COMPARISON_RELATION}, but {let_relation} is given.'
+            )
         else:
             self._comparison_relation = let_relation
 
-        self._description_pattern = ('In your response, the letter {letter} should appear {let_relation}'
-                                     ' {let_frequency} times.')
+        self._description_pattern = (
+            'In your response, the letter {letter} should appear {let_relation}'
+            ' {let_frequency} times.'
+        )
 
         return self._description_pattern.format(
             letter=self._letter,
@@ -1339,7 +1377,7 @@ class CapitalLettersEnglishChecker(Instruction):
     def check_following(self, value):
         """Checks that the response is in English and in all capital letters."""
         assert isinstance(value, str)
-
+        import langdetect
         try:
             return value.isupper() and langdetect.detect(value) == 'en'
         except langdetect.LangDetectException as e:
@@ -1353,8 +1391,10 @@ class LowercaseLettersEnglishChecker(Instruction):
 
     def build_description(self):
         """Build the instruction description."""
-        self._description_pattern = ('Your entire response should be in English, and in all lowercase'
-                                     ' letters. No capital letters are allowed.')
+        self._description_pattern = (
+            'Your entire response should be in English, and in all lowercase'
+            ' letters. No capital letters are allowed.'
+        )
         return self._description_pattern
 
     def get_instruction_args(self):
@@ -1367,7 +1407,7 @@ class LowercaseLettersEnglishChecker(Instruction):
     def check_following(self, value):
         """Checks that the response is in English and in all lowercase letters."""
         assert isinstance(value, str)
-
+        import langdetect
         try:
             return value.islower() and langdetect.detect(value) == 'en'
         except langdetect.LangDetectException as e:
@@ -1423,11 +1463,15 @@ class CapitalWordFrequencyChecker(Instruction):
         if capital_relation is None:
             self._comparison_relation = random.choice(_COMPARISON_RELATION)
         elif capital_relation not in _COMPARISON_RELATION:
-            raise ValueError('The supported relation for comparison must be in '
-                             f'{_COMPARISON_RELATION}, but {capital_relation} is given.')
-
-        self._description_pattern = ('In your response, words with all capital letters should appear'
-                                     ' {relation} {frequency} times.')
+            raise ValueError(
+                'The supported relation for comparison must be in '
+                f'{_COMPARISON_RELATION}, but {capital_relation} is given.'
+            )
+
+        self._description_pattern = (
+            'In your response, words with all capital letters should appear'
+            ' {relation} {frequency} times.'
+        )
 
         return self._description_pattern.format(frequency=self._frequency, relation=self._comparison_relation)
 

evalscope/benchmarks/ifeval/instructions_registry.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 """Registry of all instructions."""
 
-from evalscope.benchmarks.ifeval import instructions
+from . import instructions
 
 _KEYWORD = 'keywords:'
 

evalscope/benchmarks/ifeval/instructions_util.py

@@ -14,7 +14,6 @@
 """Utility library of instructions."""
 
 import functools
-import immutabledict
 import nltk
 import os
 import random
@@ -1551,7 +1550,7 @@ WORD_LIST = [
 ]  # pylint: disable=line-too-long
 
 # ISO 639-1 codes to language names.
-LANGUAGE_CODES = immutabledict.immutabledict({
+LANGUAGE_CODES = {
     'en': 'English',
     'es': 'Spanish',
     'pt': 'Portuguese',
@@ -1582,7 +1581,7 @@ LANGUAGE_CODES = immutabledict.immutabledict({
     'pa': 'Punjabi',
     'ml': 'Malayalam',
     'fi': 'Finnish',
-})
+}
 
 _ALPHABETS = '([A-Za-z])'
 _PREFIXES = '(Mr|St|Mrs|Ms|Dr)[.]'

evalscope/benchmarks/ifeval/utils.py

@@ -1,7 +1,7 @@
 import dataclasses
 from typing import Dict, Optional, Union
 
-from evalscope.benchmarks.ifeval import instructions_registry
+from . import instructions_registry
 
 
 @dataclasses.dataclass
@@ -121,14 +121,13 @@ def process_results(doc, results):
     out_loose = test_instruction_following_loose(inp, response)
 
     return {
-        'prompt_level_strict_acc': out_strict.follow_all_instructions,
-        'inst_level_strict_acc': out_strict.follow_instruction_list,
-        'prompt_level_loose_acc': out_loose.follow_all_instructions,
-        'inst_level_loose_acc': out_loose.follow_instruction_list,
+        'prompt_level_strict': float(out_strict.follow_all_instructions),
+        'inst_level_strict': agg_inst_level_acc(out_strict.follow_instruction_list),
+        'prompt_level_loose': float(out_loose.follow_all_instructions),
+        'inst_level_loose': agg_inst_level_acc(out_loose.follow_instruction_list),
     }
 
 
 def agg_inst_level_acc(items):
-    flat_items = [item for sublist in items for item in sublist]
-    inst_level_acc = sum(flat_items) / len(flat_items)
+    inst_level_acc = sum(items) / len(items) if items else 0
     return inst_level_acc