evalscope 0.17.1__py3-none-any.whl → 1.0.0__py3-none-any.whl

evalscope/benchmarks/ifeval/instructions_registry.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 """Registry of all instructions."""
 
-from evalscope.benchmarks.ifeval import instructions
+from . import instructions
 
 _KEYWORD = 'keywords:'
 

evalscope/benchmarks/ifeval/utils.py

@@ -1,7 +1,7 @@
 import dataclasses
 from typing import Dict, Optional, Union
 
-from evalscope.benchmarks.ifeval import instructions_registry
+from . import instructions_registry
 
 
 @dataclasses.dataclass
@@ -121,14 +121,13 @@ def process_results(doc, results):
     out_loose = test_instruction_following_loose(inp, response)
 
     return {
-        'prompt_level_strict_acc': out_strict.follow_all_instructions,
-        'inst_level_strict_acc': out_strict.follow_instruction_list,
-        'prompt_level_loose_acc': out_loose.follow_all_instructions,
-        'inst_level_loose_acc': out_loose.follow_instruction_list,
+        'prompt_level_strict': float(out_strict.follow_all_instructions),
+        'inst_level_strict': agg_inst_level_acc(out_strict.follow_instruction_list),
+        'prompt_level_loose': float(out_loose.follow_all_instructions),
+        'inst_level_loose': agg_inst_level_acc(out_loose.follow_instruction_list),
     }
 
 
 def agg_inst_level_acc(items):
-    flat_items = [item for sublist in items for item in sublist]
-    inst_level_acc = sum(flat_items) / len(flat_items)
+    inst_level_acc = sum(items) / len(items) if items else 0
     return inst_level_acc

evalscope/benchmarks/iquiz/iquiz_adapter.py

@@ -1,70 +1,35 @@
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
-
-
-@Benchmark.register(
-    name='iquiz',
-    pretty_name='IQuiz',
-    tags=['Knowledge', 'MCQ', 'Chinese'],
-    description=
-    'IQuiz is a benchmark for evaluating AI models on IQ and EQ questions. It consists of multiple-choice questions where the model must select the correct answer and provide an explanation.',  # noqa: E501
-    dataset_id='AI-ModelScope/IQuiz',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=['IQ', 'EQ'],
-    metric_list=['AverageAccuracy'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    system_prompt='你是一个高智商和高情商的专家，你被要求回答一个选择题，并选出一个正确的选项，解释原因，最终输出格式为：`答案是(选项)`。',  # noqa: E501
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='iquiz',
+        pretty_name='IQuiz',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE, Tags.CHINESE],
+        description=
+        'IQuiz is a benchmark for evaluating AI models on IQ and EQ questions. It consists of multiple-choice questions where the model must select the correct answer and provide an explanation.',  # noqa: E501
+        dataset_id='AI-ModelScope/IQuiz',
+        metric_list=['acc'],
+        subset_list=['IQ', 'EQ'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.CHINESE_SINGLE_ANSWER_TEMPLATE_COT,
+    )
 )
-class IQuizAdapter(DataAdapter):
+class IQuizAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.choices = ['A', 'B', 'C', 'D', 'E']
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from input data.
-        example:
-        {
-            "question":"天气预报说本周星期三会下雨，昨天果然下雨了，今天星期几？",
-            "choices":["星期一","星期二","星期三","星期四"],
-            "answer":"D",
-            "level":1
-        }
-        """
-        prompt = f"问题: {input_d['question']}\n"
-        prompt += self.__form_options(input_d['choices'])
-        return self.gen_prompt_data(prompt)
-
-    def __form_options(self, options: list):
-        option_str = '选项:\n'
-        for opt, choice in zip(options, self.choices):
-            option_str += f'({choice}): {opt}' + '\n'
-        return option_str
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-        """
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the predicted result and extract proper answer.
-        """
-        if self.model_adapter == OutputType.MULTIPLE_CHOICE:
-            return result
-        else:
-            return ResponseParser.parse_first_option_with_choices(result, self.choices)
-
-    def match(self, gold: str, pred: str) -> float:
-        """
-        Match the gold answer and the predicted answer.
-        """
-        return exact_match(gold=gold, pred=pred)
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={'level': record.get('level', 'unknown')},
+        )

evalscope/benchmarks/live_code_bench/evaluate_utils.py

@@ -130,8 +130,8 @@ def evaluate_generations(
         results[index] = result
         metadata[index] = meta
 
-    assert len(results) == len(
-        generations_list), f'results = {len(results)} inputs = {len(generations_list)} {results=}'
+    assert len(results
+               ) == len(generations_list), f'results = {len(results)} inputs = {len(generations_list)} {results=}'
 
     return results, metadata
 

evalscope/benchmarks/live_code_bench/live_code_bench_adapter.py

@@ -1,88 +1,138 @@
-from tqdm import tqdm
-
-from evalscope.benchmarks import Benchmark, DataAdapter
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import convert_numpy_types
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
 
-@Benchmark.register(
-    name='live_code_bench',
-    pretty_name='Live-Code-Bench',
-    tags=['Coding'],
-    description=
-    'Live Code Bench is a benchmark for evaluating code generation models on real-world coding tasks. It includes a variety of programming problems with test cases to assess the model\'s ability to generate correct and efficient code solutions.',  # noqa: E501
-    dataset_id='AI-ModelScope/code_generation_lite',
-    subset_list=['release_latest'],
-    metric_list=['Pass@1'],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    extra_params={
-        'start_date': None,
-        'end_date': None,
-        'timeout': 6,
-        'debug': False
-    },
-    system_prompt=
-    'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.',  # noqa: E501
-    prompt_template=
-    '### Question:\n{question_content}\n\n{format_prompt} ### Answer: (use the provided format with backticks)\n\n',  # noqa: E501
+@register_benchmark(
+    BenchmarkMeta(
+        name='live_code_bench',
+        pretty_name='Live-Code-Bench',
+        tags=[Tags.CODING],
+        description=
+        'Live Code Bench is a benchmark for evaluating code generation models on real-world coding tasks. It includes a variety of programming problems with test cases to assess the model\'s ability to generate correct and efficient code solutions.',  # noqa: E501
+        dataset_id='AI-ModelScope/code_generation_lite',
+        subset_list=['release_latest'],
+        metric_list=['Pass@1'],
+        eval_split='test',
+        prompt_template=
+        '### Question:\n{question_content}\n\n{format_prompt} ### Answer: (use the provided format with backticks)\n\n',
+        extra_params={
+            'start_date': None,
+            'end_date': None,
+            'timeout': 6,
+            'debug': False
+        },
+    )
 )
-class LiveCodeBenchAdapter(DataAdapter):
+class LiveCodeBenchAdapter(DefaultDataAdapter):
+    """
+    Live Code Bench adapter using the new data processing framework.
+    """
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        extra_params = kwargs.get('extra_params', {})
-
-        self.timeout = extra_params.get('timeout', 6)
-        self.debug = extra_params.get('debug', False)
-        self.start_date = extra_params.get('start_date')
-        self.end_date = extra_params.get('end_date')
-
-    def load(self, **kwargs) -> dict:
-        from .load_utils import filter_date, transform
-
-        # Note: need trust_remote_code=True to load the python script
-        dataset_dict = super().load(trust_remote_code=True, **kwargs)
-        new_dataset_dict = {}
-        for subset_key, dataset in dataset_dict.items():
-            datasets = dataset[self.eval_split]
-            filtered_datasets = filter_date(datasets, start_date=self.start_date, end_date=self.end_date)
-
-            transformed_datasets = [transform(item) for item in tqdm(filtered_datasets, desc='Transforming data')]
-            new_dataset_dict[subset_key] = {self.eval_split: transformed_datasets}
-        return new_dataset_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate the prompt for the model input.
-        """
-        format_prompt = input_d['format_prompt']
-        question_content = input_d['question_content']
+        self.timeout = self.extra_params.get('timeout', 6)
+        self.debug = self.extra_params.get('debug', False)
+        self.start_date = self.extra_params.get('start_date')
+        self.end_date = self.extra_params.get('end_date')
+
+        self.save_metadata = False  # Don't save metadata, since they are large
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        from .load_utils import transform
+
+        record = transform(record)
+
+        question_content = record['question_content']
+        format_prompt = record['format_prompt']
         full_prompt = self.prompt_template.format(question_content=question_content, format_prompt=format_prompt)
 
-        return self.gen_prompt_data(full_prompt)
+        return Sample(
+            input=[ChatMessageUser(content=full_prompt)],
+            target='',
+            metadata={
+                'evaluation_sample': record['evaluation_sample'],
+                'contest_date': record['contest_date']
+            }
+        )
 
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Extract the gold answer from the input dict.
-        return input_d
+    def sample_filter(self, sample):
+        from .load_utils import filter_date
 
-    def match(self, gold: dict, pred: str) -> float:
-        from .evaluate_utils import codegen_metrics
+        return filter_date(sample.metadata['contest_date'], start_date=self.start_date, end_date=self.end_date)
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        """Extract code from the prediction."""
         from .extract_utils import extract_code_generation
+        return extract_code_generation(prediction)
 
-        ext_pred = extract_code_generation(pred)
-
-        references = [{'input_output': gold['evaluation_sample']}]
-        predictions = [[ext_pred]]
-        metrics, eval_results, final_metadata = codegen_metrics(
-            references,
-            predictions,
-            k_list=[1],
-            num_process_evaluate=1,
-            timeout=self.timeout,
-            debug=self.debug,
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        from .evaluate_utils import codegen_metrics
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
         )
-        return metrics['pass@1'] / 100  # convert to point scale
+
+        references = [{'input_output': task_state.metadata['evaluation_sample']}]
+        predictions = [[filtered_prediction]]
+
+        try:
+            metrics, eval_results, final_metadata = codegen_metrics(
+                references,
+                predictions,
+                k_list=[1],
+                num_process_evaluate=1,
+                timeout=self.timeout,
+                debug=self.debug,
+            )
+            pass_rate = metrics['pass@1'] / 100  # convert to point scale
+
+            score.value = {'pass': float(pass_rate > 0)}
+            score.explanation = f"Pass@1: {metrics['pass@1']}%"
+
+            # Convert numpy types to native Python types for JSON serialization
+            serializable_eval_results = convert_numpy_types(eval_results)
+            serializable_final_metadata = convert_numpy_types(final_metadata)
+
+            score.metadata = {
+                'pass_rate': float(pass_rate),
+                'timeout': self.timeout,
+                'debug': self.debug,
+                'eval_results': serializable_eval_results,
+                'final_metadata': serializable_final_metadata
+            }
+        except Exception as e:
+            score.value = {'pass': False}
+            score.explanation = f'Evaluation failed: {str(e)}'
+            score.metadata = {'error': str(e)}
+
+        score.main_score_name = 'pass'
+        return score
+
+    def aggregate_scores(self, sample_scores):
+        from evalscope.metrics.metric import PassAtK
+
+        # calculate pass@k here
+        agg_list = []
+        for metric in self.metric_list:
+            if metric.lower().startswith('pass@'):
+                k = int(metric.split('@')[1])
+                # Get the scores for this metric
+                agg = PassAtK(k)
+                agg_list.extend(agg(sample_scores))
+        return agg_list

evalscope/benchmarks/live_code_bench/load_utils.py

@@ -32,8 +32,8 @@ def transform(item):
         private_test_cases = json.loads(item['private_test_cases'])
     except Exception as e:  # noqa: F841
         private_test_cases = json.loads(
-            pickle.loads(zlib.decompress(base64.b64decode(private_test_cases.encode('utf-8'))  # type: ignore
-                                         )))  # type: ignore
+            pickle.loads(zlib.decompress(base64.b64decode(private_test_cases.encode('utf-8'))))
+        )
 
     # load metadata
     metadata = json.loads(item['metadata'])
@@ -47,25 +47,17 @@ def transform(item):
     return item
 
 
-def filter_date(dataset, start_date=None, end_date=None):
-    new_dataset = []
-
-    for item in dataset:
-        contest_date = datetime.fromisoformat(item['contest_date'])
-        if start_date is not None:
-            p_start_date = datetime.strptime(start_date, '%Y-%m-%d')
-            if p_start_date > contest_date:
-                continue
+def filter_date(contest_date, start_date=None, end_date=None) -> bool:
 
-        if end_date is not None:
-            p_end_date = datetime.strptime(end_date, '%Y-%m-%d')
-            if p_end_date < contest_date:
-                continue
+    contest_date = datetime.fromisoformat(contest_date)
+    if start_date is not None:
+        p_start_date = datetime.strptime(start_date, '%Y-%m-%d')
+        if p_start_date > contest_date:
+            return False
 
-        new_dataset.append(item)
+    if end_date is not None:
+        p_end_date = datetime.strptime(end_date, '%Y-%m-%d')
+        if p_end_date < contest_date:
+            return False
 
-    if start_date or end_date:
-        logger.info(
-            f'Filtered dataset with start_date: {start_date}, end_date: {end_date}, remaining items: {len(new_dataset)}'
-        )
-    return new_dataset
+    return True

evalscope/benchmarks/live_code_bench/testing_util.py

@@ -4,18 +4,22 @@ import faulthandler
 import json
 import numpy as np
 import platform
+
 # to run the solution files we're using a timing based approach
 import signal
 import sys
 import time
+
 # used for debugging to time steps
 from datetime import datetime
 from decimal import Decimal
 from enum import Enum
 from functools import partial
 from io import StringIO
+
 # from pyext import RuntimeModule
 from types import ModuleType
+
 # used for testing the code that reads from input
 from unittest.mock import mock_open, patch
 
@@ -342,8 +346,8 @@ def grade_stdio(
             return all_results, WA_send_args
 
         for output_line_idx, (
-                stripped_prediction_line,
-                stripped_gt_out_line,
+            stripped_prediction_line,
+            stripped_gt_out_line,
         ) in enumerate(zip(stripped_prediction_lines, stripped_gt_out_lines)):
             WA_send_args['error_message'] = (
                 f'Wrong answer at {output_line_idx=}: {truncatefn(stripped_prediction_line)} != {truncatefn(stripped_gt_out_line)}'

evalscope/benchmarks/maritime_bench/maritime_bench_adapter.py

@@ -1,82 +1,56 @@
 from typing import Any
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import EvalType, OutputType
-from evalscope.metrics import exact_match
-from evalscope.metrics.completion_parsers import ResponseParser
-
-SUBSET_LIST = ['default']
-
-
-@Benchmark.register(
-    name='maritime_bench',
-    pretty_name='MaritimeBench',
-    tags=['Maritime', 'MCQ', 'Knowledge'],
-    description=
-    'MaritimeBench is a benchmark for evaluating AI models on maritime-related multiple-choice questions. It consists of questions related to maritime knowledge, where the model must select the correct answer from given options.',  # noqa: E501
-    dataset_id='HiDolphin/MaritimeBench',
-    model_adapter=OutputType.GENERATION,
-    output_types=[OutputType.MULTIPLE_CHOICE, OutputType.GENERATION],
-    subset_list=SUBSET_LIST,
-    metric_list=['AverageAccuracy'],
-    eval_split='test',
-    prompt_template=
-    '题目来自于{subset_name}请回答单选题。要求只输出选项，不输出解释，将选项放在<>里，直接输出答案。示例：\n\n题目：在船舶主推进动力装置中，传动轴系在运转中承受以下复杂的应力和负荷，但不包括______。\n选项：\nA. 电磁力\nB. 压拉应力\nC. 弯曲应力\nD. 扭应力\n答：<A> 当前题目\n {query}',  # noqa: E501
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+
+MARITIME_PROMPT_TEMPLATE = '请回答单选题。要求只输出选项，不输出解释，将选项放在[]里，直接输出答案。示例：\n\n题目：在船舶主推进动力装置中，传动轴系在运转中承受以下复杂的应力和负荷，但不包括______。\n选项：\nA. 电磁力\nB. 压拉应力\nC. 弯曲应力\nD. 扭应力\n答：[A]\n 当前题目\n {question}\n选项：\n{choices}'  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='maritime_bench',
+        pretty_name='MaritimeBench',
+        tags=[Tags.CHINESE, Tags.MULTIPLE_CHOICE, Tags.KNOWLEDGE],
+        description=
+        'MaritimeBench is a benchmark for evaluating AI models on maritime-related multiple-choice questions. It consists of questions related to maritime knowledge, where the model must select the correct answer from given options.',  # noqa: E501
+        dataset_id='HiDolphin/MaritimeBench',
+        metric_list=['acc'],
+        few_shot_num=0,
+        eval_split='test',
+        prompt_template=MARITIME_PROMPT_TEMPLATE,
+    )
 )
-class MaritimeBenchAdapter(DataAdapter):
+class MaritimeBenchAdapter(MultiChoiceAdapter):
 
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.choices = ['A', 'B', 'C', 'D']
-
-    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> Any:
-
-        prefix = ''
-        query = prefix + input_d['question'] + '\n'
-        available_choices = []
-        for option in self.choices:
-            if option in input_d and input_d[option]:
-                query += option + ':' + input_d[option] + '\n'
-                available_choices.append(option)
-
-        full_prompt = self.prompt_template.format(subset_name=subset_name, query=query)
-        return self.gen_prompt_data(full_prompt, choices=available_choices)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        """
-        Parse the raw input labels (gold).
-
-        Args:
-            input_d: input raw data. Depending on the dataset.
-
-        Returns:
-            The parsed input. e.g. gold answer ... Depending on the dataset.
-        """
-        return input_d['answer']
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
-        """
-        Parse the raw model prediction (pred).
-
-        Args:
-            pred: model prediction. Depending on the model.
-
-        Returns:
-            The parsed prediction. e.g. model answer... Depending on the model.
-        """
-
-        return ResponseParser.parse_bracketed_answer(result, options=self.choices)
-
-    def match(self, gold: Any, pred: Any) -> Any:
-        """
-        Match the gold answer with the predicted answer.
-
-        Args:
-            gold: The gold answer.
-            pred: The predicted answer.
-
-        Returns:
-            The result of the match.
-        """
-        return exact_match(gold=gold, pred=pred)
+        self.reformat_subset = True
+
+    def record_to_sample(self, record) -> Sample:
+        # Extract available choices from the record
+        choices = []
+        choice_letters = ['A', 'B', 'C', 'D']
+        for letter in choice_letters:
+            if letter in record and record[letter]:
+                choices.append(record[letter])
+
+        return Sample(
+            input=record['question'],
+            choices=choices,
+            target=record['answer'],
+        )
+
+    def format_prompt_template(self, sample):
+        choices = '\n'.join([f'{chr(65 + i)}. {choice}' for i, choice in enumerate(sample.choices)])
+        return MARITIME_PROMPT_TEMPLATE.format(question=sample.input, choices=choices)
+
+    def extract_answer(self, prediction, task_state):
+        # use regex to extract the answer from the prediction
+        import re
+        match = re.search(r'\[([A-D])\]', prediction)
+        if match:
+            return match.group(1)
+        return ''