evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/piqa/piqa_adapter.py

@@ -0,0 +1,32 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+DESCRIPTION = 'PIQA addresses the challenging task of reasoning about physical commonsense in natural language.'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='piqa',
+        pretty_name='PIQA',
+        tags=[Tags.REASONING, Tags.COMMONSENSE, Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/piqa',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split='train',
+        eval_split='validation',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
+)
+class PIQAAdapter(MultiChoiceAdapter):
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )

evalscope/benchmarks/poly_math/poly_math_adapter.py

@@ -0,0 +1,132 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+
+from typing import Any, Dict, List, Tuple
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.dataset.dataset import DatasetDict, MemoryDataset
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.report.report import Report, Subset
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'en', 'zh', 'ar', 'bn', 'de', 'es', 'fr', 'id', 'it', 'ja', 'ko', 'ms', 'pt', 'ru', 'sw', 'te', 'th', 'vi'
+]
+LEVEL_LIST = ['low', 'medium', 'high', 'top']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='poly_math',
+        pretty_name='PolyMath',
+        tags=[Tags.MATH, Tags.REASONING, Tags.MULTI_LINGUAL],
+        description=
+        'PolyMath is a multilingual mathematical reasoning benchmark covering 18 languages and 4 easy-to-hard difficulty levels, with 9,000 high-quality problem samples. Our benchmark ensures difficulty comprehensiveness, language diversity, and high-quality translation, making it a highly discriminative multilingual mathematical benchmark in the era of reasoning LLMs.',  # noqa: E501
+        dataset_id='evalscope/PolyMath',
+        subset_list=SUBSET_LIST,
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        eval_split='test',
+        prompt_template='{question}',
+    )
+)
+class PolyMathAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+    def load(self) -> Tuple[DatasetDict, None]:
+        """Load all difficulty levels, rename subsets with their level suffix, and merge them."""
+        # Need to load all levels to get the full dataset
+        dataset_list: List[Dict[str, MemoryDataset]] = []
+        original_split = getattr(self, 'eval_split', None)
+        try:
+            for level in LEVEL_LIST:
+                self.eval_split = level
+                cur_level_dataset_dict, _ = super().load()
+                # Build a renamed mapping without mutating during iteration
+                renamed: Dict[str, MemoryDataset] = {
+                    f'{subset}-{level}': ds
+                    for subset, ds in cur_level_dataset_dict.items()
+                }
+                dataset_list.append(renamed)
+        finally:
+            # Restore original split to avoid side effects
+            if original_split is not None:
+                self.eval_split = original_split
+        # Merge all levels into one dataset
+        return DatasetDict.from_dataset_dicts(dataset_list), None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a single record into a Sample with language-specific instruction."""
+        from .utils.instruction import query_dic
+
+        # e.g., 'high-en-1'
+        question_id = record['id']
+        level, language, index = question_id.split('-')
+        # Get the instruction for the specific language
+        instruction = query_dic[language]
+
+        return Sample(
+            input=record['question'] + '\n' + instruction,
+            target=record['answer'],
+            metadata={
+                'level': level,
+                'language': language,
+                'index': index,
+            },
+        )
+
+    def extract_answer(self, prediction: str, task_state):
+        from evalscope.metrics.math_parser import extract_answer
+
+        return extract_answer(prediction)
+
+    def _on_generate_report_end(self, report: Report, output_dir, **kwargs):
+        """
+        Finalize the report generation process. Calculate the difficulty-weighted accuracy (DW-ACC)
+        per language and the overall DW-ACC, and append as a new category to each metric.
+        """
+        from evalscope.report import Category, Metric
+
+        WEIGHT_DENOMINATOR = 15  # 1 + 2 + 4 + 8 for ['low','medium','high','top']
+
+        for metric in report.metrics:
+            # Collect all subsets by name for easy lookup (e.g., "en-low")
+            subset_dict: Dict[str, Subset] = {}
+            for category in metric.categories:
+                for subset in category.subsets:
+                    subset_dict[subset.name] = subset
+
+            # Compute per-language DW-ACC
+            dw_subsets: List[Subset] = []
+            for language in SUBSET_LIST:
+                weighted_sum = 0.0
+                total_num = 0
+                for i, level in enumerate(LEVEL_LIST):
+                    s = subset_dict.get(f'{language}-{level}')
+                    if s is not None:
+                        weighted_sum += (2**i) * s.score
+                        total_num += s.num
+                # missing subsets contribute 0 by design
+                if total_num == 0:
+                    continue
+                dw_acc = weighted_sum / WEIGHT_DENOMINATOR
+                dw_subsets.append(Subset(name=language, score=dw_acc, num=total_num))
+
+            # Overall DW-ACC: unweighted average across languages
+            if dw_subsets:
+                overall_score = sum(s.score for s in dw_subsets) / len(dw_subsets)
+                overall_num = sum(s.num for s in dw_subsets)
+                dw_subsets.append(Subset(name='OVERALL', score=overall_score, num=overall_num))
+
+        # Append DW-ACC metric
+        if dw_subsets:
+            report.metrics.append(Metric(name='DW-ACC', categories=[Category(name='-', subsets=dw_subsets)]))

evalscope/benchmarks/poly_math/utils/instruction.py

@@ -0,0 +1,105 @@
+# flake8: noqa
+language_dic = {
+    'en': 'English',
+    'zh': 'Chinese',
+    'ar': 'Arabic',
+    'bn': 'Bangali',
+    'de': 'German',
+    'es': 'Spanish',
+    'fr': 'French',
+    'id': 'Indonesian',
+    'it': 'Italian',
+    'ja': 'Japanese',
+    'ko': 'Korean',
+    'ms': 'Malay',
+    'pt': 'Potuguese',
+    'ru': 'Russian',
+    'sw': 'Swahili',
+    'te': 'Telugu',
+    'th': 'Thai',
+    'vi': 'Vietnamese',
+}
+
+query_dic = {
+    'en': 'Note: Please put the final answer in the $\\boxed\{\}$.',
+    'zh': '注意：请将最终答案放在 $\\boxed\{\}$ 中。',
+    'ar': 'ملاحظة: يُرجى وضع الإجابة النهائية في $\\boxed\{\}$.',
+    'bn': 'বিঃদ্রঃ: অনুগ্রহ করে চূড়ান্ত উত্তরটি $\\boxed\{\}$ এর মধ্যে রাখুন।',
+    'de': 'Hinweis: Bitte setzen Sie die endgültige Antwort in $\\boxed\{\}$.',
+    'es': 'Nota: Por favor, coloque la respuesta final en el $\\boxed\{\}$.',
+    'fr': 'Remarque : Veuillez mettre la réponse finale dans le $\\boxed\{\}$.',
+    'id': 'Catatan: Silakan letakkan jawaban akhir di dalam $\\boxed\{\}$.',
+    'it': 'Nota: Per favore, metti la risposta finale nel $\\boxed\{\}$.',
+    'ja': '注意：最終的な答えを $\\boxed\{\}$ に入れてください。',
+    'ko': '참고: 최종 답안을 $\\boxed\{\}$ 안에 넣어 주세요.',
+    'ms': 'Nota: Sila letakkan jawapan akhir dalam $\\boxed\{\}$.',
+    'pt': 'Nota: Por favor, coloque a resposta final no $\\boxed\{\}$.',
+    'ru': 'Примечание: Пожалуйста, поместите окончательный ответ в $\\boxed\{\}$.',
+    'sw': 'Kumbuka: Tafadhali weka jibu la mwisho katika $\\boxed\{\}$.',
+    'te': 'గమనిక: దయచేసి తుది జవాబును $\\boxed\{\}$ లో ఉంచండి.',
+    'th': 'หมายเหตุ: กรุณาใส่คำตอบสุดท้ายใน $\\boxed\{\}$.',
+    'vi': 'Lưu ý: Vui lòng đặt câu trả lời cuối cùng trong $\\boxed\{\}$.',
+}
+
+language_control = {
+    'forcing_raw': {
+        'en': 'Use English to think and answer.',
+        'zh': '使用中文进行思考和回答。',
+        'ar': 'استخدم العربية للتفكير والإجابة.',
+        'bn': 'বাংলা ব্যবহার করে চিন্তা এবং উত্তর দিন।',
+        'de': 'Verwende Deutsch, um zu denken und zu antworten.',
+        'es': 'Usa español para pensar y responder.',
+        'fr': 'Utilisez le français pour penser et répondre.',
+        'id': 'Gunakan bahasa Indonesia untuk berpikir dan menjawab.',
+        'it': 'Usa italiano per pensare e rispondere.',
+        'ja': '日本語を使って考え、回答してください。',
+        'ko': '한국어로 생각하고 답변하세요.',
+        'ms': 'Gunakan bahasa Melayu untuk berfikir dan menjawab.',
+        'pt': 'Use português para pensar e responder.',
+        'ru': 'Используйте русский язык для размышлений и ответов.',
+        'sw': 'Tumia Kiswahili kufikiri na kujibu.',
+        'te': 'తెలుగును ఉపయోగించి ఆలోచించి సమాధానం ఇవ్వండి.',
+        'th': 'ใช้ภาษาไทยในการคิดและตอบคำถาม.',
+        'vi': 'Sử dụng tiếng Việt để suy nghĩ và trả lời.',
+    },
+    'forcing_en': {
+        'en': 'Use English to think and answer.',
+        'zh': '使用英文进行思考和回答。',
+        'ar': 'استخدم اللغة الإنجليزية للتفكير والإجابة.',
+        'bn': 'ইংরেজি ব্যবহার করে চিন্তা এবং উত্তর দিন।',
+        'de': 'Verwenden Sie Englisch, um zu denken und zu antworten.',
+        'es': 'Usa inglés para pensar y responder.',
+        'fr': "Utilisez l'anglais pour penser et répondre.",
+        'id': 'Gunakan bahasa Inggris untuk berpikir dan menjawab.',
+        'it': 'Usa inglese per pensare e rispondere.',
+        'ja': '英語を使って考え、回答してください。',
+        'ko': '영어로 생각하고 답변하세요.',
+        'ms': 'Gunakan bahasa Inggeris untuk berfikir dan menjawab.',
+        'pt': 'Use inglês para pensar e responder.',
+        'ru': 'Используйте английский язык, чтобы думать и отвечать.',
+        'sw': 'Tumia Kiingereza kufikiri na kujibu.',
+        'te': 'ఇంగ్లీష్‌ను ఉపయోగించి ఆలోచించి ఉత్తరించండి.',
+        'th': 'ใช้ภาษาอังกฤษในการคิดและตอบคำถาม.',
+        'vi': 'Sử dụng tiếng Anh để suy nghĩ và trả lời.',
+    },
+    'forcing_prefer': {
+        'en': 'Choose the language you are most proficient in to think and answer.',
+        'zh': '自选一种你最擅长的语言进行思考和回答。',
+        'ar': 'اختر اللغة التي تجيدها أكثر للتفكير والإجابة.',
+        'bn': 'আপনি যে ভাষাটি সবচেয়ে পারদর্শী সেটি বেছে নিয়ে চিন্তা এবং উত্তর দিন।',
+        'de': 'Wählen Sie die Sprache, in der Sie am kompetentesten sind, um zu denken und zu antworten.',
+        'es': 'Elige el idioma en el que eres más competente para pensar y responder.',
+        'fr': 'Choisissez la langue dans laquelle vous êtes le plus compétent pour penser et répondre.',
+        'id': 'Pilih bahasa yang paling Anda kuasai untuk berpikir dan menjawab.',
+        'it': 'Scegli la lingua in cui sei più competente per pensare e rispondere.',
+        'ja': '最も得意な言語を選んで考え、回答してください。',
+        'ko': '가장 능숙한 언어를 선택하여 생각하고 답변하세요.',
+        'ms': 'Pilih bahasa yang paling anda mahir untuk berfikir dan menjawab.',
+        'pt': 'Escolha o idioma em que você é mais competente para pensar e responder.',
+        'ru': 'Выберите язык, в котором вы наиболее компетентны, чтобы думать и отвечать.',
+        'sw': 'Chagua lugha ambayo unamudu zaidi kufikiri na kujibu.',
+        'te': 'మీరు అత్యంత స్థిరంగా ఉన్న భాషను స్వీకరించి ఆలోచించిตอบ。',
+        'th': 'เลือกภาษาที่คุณมีความสามารถมากที่สุดในการคิดและตอบคำถาม.',
+        'vi': 'Chọn ngôn ngữ mà bạn thành thạo nhất để suy nghĩ và trả lời.',
+    }
+}

evalscope/benchmarks/pope/pope_adapter.py

@@ -0,0 +1,112 @@
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='pope',
+        pretty_name='POPE',
+        tags=[Tags.MULTI_MODAL, Tags.HALLUCINATION, Tags.YES_NO],
+        description=
+        'POPE (Polling-based Object Probing Evaluation) is a benchmark designed to evaluate object hallucination in large vision-language models (LVLMs). It tests models by having them answer simple yes/no questions about the presence of specific objects in an image. This method helps measure how accurately a model\'s responses align with the visual content, with a focus on identifying instances where models claim objects exist that are not actually present. The benchmark employs various sampling strategies, including random, popular, and adversarial sampling, to create a robust set of questions for assessment.',  # noqa: E501
+        dataset_id='lmms-lab/POPE',
+        metric_list=['accuracy', 'precision', 'recall', 'f1_score', 'yes_ratio'],
+        aggregation='f1',
+        subset_list=['popular', 'adversarial', 'random'],
+        default_subset='Full',
+        prompt_template='{question}\nPlease answer YES or NO without an explanation.',
+    )
+)
+class PopeAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.split_as_subset = True
+        self.add_overall_metric = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+
+        input_text = self.prompt_template.format(question=record['question'])
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        answer = record['answer'].upper()  # 'YES' or 'NO'
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=answer,
+            metadata={
+                'id': record.get('id'),
+                'answer': answer,
+                'category': record.get('category'),
+                'question_id': record.get('question_id'),
+            }
+        )
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        # Check if the reference answer is in the filtered prediction
+        result = 1 if reference in filtered_prediction.strip().upper() else 0
+        score.value = {'acc': result}
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Custom aggregation to compute accuracy, precision, recall, f1_score, and yes_ratio.
+        """
+
+        def compute_metrics(scores: List[SampleScore]):
+            tp = fp = tn = fn = 0
+            yes_count = 0
+            total_count = len(scores)
+
+            for ss in scores:
+                gt = ss.sample_metadata['answer'].strip().upper()
+                # Get prediction based on score
+                pred = gt if ss.score.main_value == 1 else ('NO' if gt == 'YES' else 'YES')
+                if pred == 'YES':
+                    yes_count += 1
+                if pred == 'YES' and gt == 'YES':
+                    tp += 1
+                elif pred == 'YES' and gt == 'NO':
+                    fp += 1
+                elif pred == 'NO' and gt == 'NO':
+                    tn += 1
+                elif pred == 'NO' and gt == 'YES':
+                    fn += 1
+
+            accuracy = (tp + tn) / total_count if total_count > 0 else 0.0
+            precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+            recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+            f1_score = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
+            yes_ratio = yes_count / total_count if total_count > 0 else 0.0
+
+            return {
+                'accuracy': accuracy,
+                'precision': precision,
+                'recall': recall,
+                'f1_score': f1_score,
+                'yes_ratio': yes_ratio
+            }
+
+        overall_metrics = compute_metrics(sample_scores)
+        agg_scores = []
+        for metric_name, value in overall_metrics.items():
+            agg_scores.append(AggScore(metric_name=metric_name, score=value, num=len(sample_scores), metadata={}))
+
+        return agg_scores

evalscope/benchmarks/process_bench/process_bench_adapter.py

@@ -39,6 +39,7 @@ Please put your final answer (i.e., the index) in \boxed{{}}.
         dataset_id='Qwen/ProcessBench',
         subset_list=['gsm8k', 'math', 'olympiadbench', 'omnimath'],
         metric_list=['error_acc', 'correct_acc', 'simple_f1_score'],
+        aggregation='f1',
         eval_split='test',
         prompt_template=CRITIQUE_TEMPLATE
     )

evalscope/benchmarks/pumed_qa/pubmed_qa_adapter.py

@@ -0,0 +1,175 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DESCRIPTION = ('PubMedQA reasons over biomedical research texts to answer the multiple-choice questions.')
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='pubmedqa',
+        pretty_name='PubMedQA',
+        tags=[Tags.KNOWLEDGE, Tags.YES_NO],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/pubmed-qa',
+        metric_list=['accuracy', 'precision', 'recall', 'f1_score', 'yes_ratio', 'maybe_ratio'],
+        aggregation='f1',
+        few_shot_num=0,
+        eval_split='test',
+        prompt_template='{question}\nPlease answer YES or NO or MAYBE without an explanation.',
+    )
+)
+class PubMedQAAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_overall_metric = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        abstract = record['context']
+        question = record['question']
+        question = f'Abstract: {abstract}\n\nQuestion: {question}'
+        input_text = self.prompt_template.format(question=question)
+        content_list: List[Content] = [ContentText(text=input_text)]
+        answer = str(record['answer']).upper()  # 'YES' or 'NO' or 'MAYBE'
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=answer,
+            metadata={
+                'answer': record['answer'],
+                'reasoning': record['reasoning'],
+            }
+        )
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        # Check if the reference answer is in the filtered prediction
+        result = 1 if re.search(r'\b' + re.escape(reference) + r'\b', filtered_prediction.strip().upper()) else 0
+        score.value = {'acc': result}
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Custom aggregation to compute accuracy, precision, recall, f1_score, yes_ratio and maybe_ratio.
+        Handles multi-class classification with YES, NO, and MAYBE answers.
+        """
+
+        def compute_metrics(scores: List[SampleScore]):
+            # Initialize confusion matrix for multi-class classification
+            confusion_matrix = {
+                'YES': {
+                    'YES': 0,
+                    'NO': 0,
+                    'MAYBE': 0
+                },
+                'NO': {
+                    'YES': 0,
+                    'NO': 0,
+                    'MAYBE': 0
+                },
+                'MAYBE': {
+                    'YES': 0,
+                    'NO': 0,
+                    'MAYBE': 0
+                }
+            }
+
+            yes_count = 0
+            maybe_count = 0
+            total_count = len(scores)
+            correct_count = 0
+
+            for ss in scores:
+                gt = ss.sample_metadata['answer'].strip().upper()
+
+                if ss.score.main_value == 1:
+                    correct_count += 1
+                    pred = gt
+                else:
+                    pred_text = ss.score.extracted_prediction.strip().upper()
+                    # Heuristic to determine the predicted class from text
+                    if 'YES' in pred_text:
+                        pred = 'YES'
+                    elif 'NO' in pred_text:
+                        pred = 'NO'
+                    elif 'MAYBE' in pred_text:
+                        pred = 'MAYBE'
+                    else:
+                        pred = None
+
+                if pred:
+                    if pred == 'YES':
+                        yes_count += 1
+                    elif pred == 'MAYBE':
+                        maybe_count += 1
+
+                    if gt in confusion_matrix and pred in confusion_matrix[gt]:
+                        confusion_matrix[gt][pred] += 1
+
+            # Calculate accuracy
+            accuracy = correct_count / total_count if total_count > 0 else 0.0
+
+            # Calculate per-class precision, recall, and F1
+            classes = ['YES', 'NO', 'MAYBE']
+            precision_values = []
+            recall_values = []
+            f1_values = []
+
+            for cls in classes:
+                # True positives for this class
+                tp = confusion_matrix[cls][cls]
+
+                # Calculate predicted positives (column sum)
+                pred_pos = sum(confusion_matrix[true_cls][cls] for true_cls in classes)
+
+                # Calculate actual positives (row sum)
+                act_pos = sum(confusion_matrix[cls][pred_cls] for pred_cls in classes)
+
+                # Calculate precision and recall for this class
+                cls_precision = tp / pred_pos if pred_pos > 0 else 0.0
+                cls_recall = tp / act_pos if act_pos > 0 else 0.0
+
+                # Calculate F1 for this class
+                cls_f1 = (2 * cls_precision * cls_recall) / (cls_precision
+                                                             + cls_recall) if (cls_precision + cls_recall) > 0 else 0.0
+
+                precision_values.append(cls_precision)
+                recall_values.append(cls_recall)
+                f1_values.append(cls_f1)
+
+            # Macro average (simple average across all classes)
+            precision = sum(precision_values) / len(precision_values) if precision_values else 0.0
+            recall = sum(recall_values) / len(recall_values) if recall_values else 0.0
+            f1_score = sum(f1_values) / len(f1_values) if f1_values else 0.0
+
+            # Calculate ratios
+            yes_ratio = yes_count / total_count if total_count > 0 else 0.0
+            maybe_ratio = maybe_count / total_count if total_count > 0 else 0.0
+
+            return {
+                'accuracy': accuracy,
+                'precision': precision,
+                'recall': recall,
+                'f1_score': f1_score,
+                'yes_ratio': yes_ratio,
+                'maybe_ratio': maybe_ratio
+            }
+
+        overall_metrics = compute_metrics(sample_scores)
+        agg_scores = []
+        for metric_name, value in overall_metrics.items():
+            agg_scores.append(AggScore(metric_name=metric_name, score=value, num=len(sample_scores), metadata={}))
+
+        return agg_scores

evalscope/benchmarks/qasc/qasc_adapter.py

@@ -0,0 +1,35 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+DESCRIPTION = (
+    'QASC is a question-answering dataset with a focus on sentence composition. '
+    'It consists of 9,980 8-way multiple-choice questions about grade school science.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='qasc',
+        pretty_name='QASC',
+        tags=[Tags.KNOWLEDGE, Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/qasc',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='validation',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
+)
+class QASCAdapter(MultiChoiceAdapter):
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )