evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/coin_flip/coin_flip_adapter.py

@@ -0,0 +1,128 @@
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DESCRIPTION = (
+    "CoinFlip is a symbolic reasoning dataset that tests an LLM's ability "
+    'to track binary state changes through a sequence of actions. '
+    'Each example describes whether a coin is flipped or not by different person, '
+    'requiring logical inference to determine the final state (heads or tails).'
+)  # noqa: E501
+
+PROMPT_TEMPLATE = """
+Solve the following coin flip problem step by step. The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the problem.
+
+{question}
+
+Remember to put your answer on its own line at the end in the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer YES or NO to the problem.
+
+Reasoning:
+"""  # noqa: E501
+
+FEWSHOT_TEMPLATE = """
+Here are some examples of how to solve similar problems:
+
+{fewshot}
+
+""".lstrip() + PROMPT_TEMPLATE  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='coin_flip',
+        pretty_name='CoinFlip',
+        tags=[Tags.REASONING, Tags.YES_NO],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/coin-flip',
+        metric_list=['accuracy', 'precision', 'recall', 'f1_score', 'yes_ratio'],
+        aggregation='f1',
+        few_shot_num=0,
+        train_split='validation',
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
+)
+class CoinFlipAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_overall_metric = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        question = record['question']
+        answer = record['answer']
+        input_text = self.prompt_template.format(question=question)
+        content_list: List[Content] = [ContentText(text=input_text)]
+        answer = str(answer).upper()  # 'YES' or 'NO'
+        return Sample(input=[ChatMessageUser(content=content_list)], target=answer, metadata={
+            'answer': answer,
+        })
+
+    def extract_answer(self, prediction, task_state):
+        import re
+
+        match = re.search(r'ANSWER:\s*(.*)', prediction)
+        return match.group(1) if match else prediction
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        # Check for an exact match against the extracted answer.
+        result = 1 if reference in filtered_prediction else 0
+        score.value = {'acc': result}
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Custom aggregation to compute accuracy, precision, recall, f1_score, and yes_ratio.
+        """
+
+        tp = fp = tn = fn = 0
+        yes_count = 0
+        total_count = len(sample_scores)
+
+        for ss in sample_scores:
+            gt = ss.sample_metadata['answer'].strip().upper()
+            pred = ss.score.extracted_prediction.strip().upper()
+
+            if pred == 'YES':
+                yes_count += 1
+            if pred == 'YES' and gt == 'YES':
+                tp += 1
+            elif pred == 'YES' and gt == 'NO':
+                fp += 1
+            elif pred == 'NO' and gt == 'NO':
+                tn += 1
+            elif pred == 'NO' and gt == 'YES':
+                fn += 1
+
+        accuracy = (tp + tn) / total_count if total_count > 0 else 0.0
+        precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+        recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+        f1_score = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
+        yes_ratio = yes_count / total_count if total_count > 0 else 0.0
+
+        overall_metrics = {
+            'accuracy': accuracy,
+            'precision': precision,
+            'recall': recall,
+            'f1_score': f1_score,
+            'yes_ratio': yes_ratio
+        }
+
+        agg_scores = []
+        for metric_name, value in overall_metrics.items():
+            agg_scores.append(AggScore(metric_name=metric_name, score=value, num=len(sample_scores), metadata={}))
+
+        return agg_scores

evalscope/benchmarks/commonsense_qa/commonsense_qa_adapter.py

@@ -0,0 +1,32 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+DESCRIPTION = 'CommonsenseQA requires different types of commonsense knowledge to predict the correct answers.'
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='commonsense_qa',
+        pretty_name='CommonsenseQA',
+        tags=[Tags.REASONING, Tags.COMMONSENSE, Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/commonsense-qa',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='validation',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER,
+    )
+)
+class CommonsenseQAAdapter(MultiChoiceAdapter):
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )

evalscope/benchmarks/competition_math/competition_math_adapter.py

@@ -71,3 +71,8 @@ class CompetitionMathAdapter(DefaultDataAdapter):
 
     def sample_to_fewshot(self, sample: Sample) -> str:
         return f'Problem:\n{sample.input}\nSolution:\n{sample.target}'
+
+    def extract_answer(self, prediction: str, task_state):
+        from evalscope.metrics.math_parser import extract_answer
+
+        return extract_answer(prediction)

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -6,9 +6,7 @@ from typing import Any, Dict, List
 from evalscope.api.benchmark import BenchmarkMeta, DataAdapter, DefaultDataAdapter
 from evalscope.api.dataset import DatasetDict, LocalDataLoader, Sample
 from evalscope.api.evaluator import TaskState
-from evalscope.api.metric import Score
 from evalscope.api.metric.scorer import AggScore, SampleScore
-from evalscope.api.model.model import Model
 from evalscope.api.registry import get_benchmark, register_benchmark
 from evalscope.config import TaskConfig
 from evalscope.constants import DataCollection, Tags
@@ -22,8 +20,13 @@ logger = get_logger()
 @register_benchmark(
     BenchmarkMeta(
         name=DataCollection.NAME,
+        pretty_name='Data-Collection',
         dataset_id='',  # dataset_id need to be set
-        description='Data collection',
+        description='Custom Data collection, mixing multiple evaluation datasets for '
+        'a unified evaluation, aiming to use less data to achieve a more comprehensive '
+        'assessment of the model\'s capabilities. '
+        '[Usage Reference](https://evalscope.readthedocs.io/en/latest/advanced_guides/collection/index.html)',
+        tags=[Tags.CUSTOM],
         metric_list=['acc'],
         eval_split='test',
         prompt_template='',
@@ -55,9 +58,10 @@ class DataCollectionAdapter(DefaultDataAdapter):
             data_id_or_path=dataset_path,
             split=self.eval_split,
             sample_fields=self.record_to_sample,
-            subset=self.default_subset,
+            subset='test',  # NOTE: using hardcoded test subset
             limit=self.limit,
-            repeats=self.repeats
+            repeats=self.repeats,
+            shuffle=self.shuffle,
         ).load()
 
         test_dataset = DatasetDict({self.default_subset: dataset})
@@ -95,7 +99,6 @@ class DataCollectionAdapter(DefaultDataAdapter):
 
         # load dataset args
         dataset_args = copy.deepcopy(self._task_config.dataset_args)
-        common_args = dataset_args.get(DataCollection.NAME, {})
 
         # Iterate through each sample in the dataset
         dataset = self.test_dataset[self.default_subset]
@@ -108,7 +111,6 @@ class DataCollectionAdapter(DefaultDataAdapter):
 
             # update dataset args
             cur_dataset_args = dataset_args.get(dataset_name, {})
-            cur_dataset_args.update(common_args)
 
             # Initialize dataset adapter
             if dataset_name not in self.dataset_adapters:
@@ -141,19 +143,22 @@ class DataCollectionAdapter(DefaultDataAdapter):
         data = []
         for sample_score in sample_scores:
             collection_info = sample_score.sample_metadata[DataCollection.INFO]
-            for metric_name, value in sample_score.score.value.items():
-                data.append(
-                    dict(
-                        task_type=collection_info['task_type'],
-                        categories=tuple(collection_info['categories']),
-                        dataset_name=collection_info['dataset_name'],
-                        subset_name=collection_info['subset_name'],
-                        tags=collection_info['tags'],
-                        sample_id=sample_score.sample_id,
-                        metric=metric_name,
-                        score=value
-                    )
+            main_score = sample_score.score.main_value
+            main_metric = sample_score.score.main_score_name
+
+            # use main score
+            data.append(
+                dict(
+                    task_type=collection_info['task_type'],
+                    categories=tuple(collection_info['categories']),
+                    dataset_name=collection_info['dataset_name'],
+                    subset_name=collection_info['subset_name'],
+                    tags=collection_info['tags'],
+                    sample_id=sample_score.sample_id,
+                    metric=main_metric,
+                    score=main_score
                 )
+            )
 
         df = pd.DataFrame(data)
 

evalscope/benchmarks/docvqa/docvqa_adapter.py

@@ -0,0 +1,67 @@
+import json
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+PROMPT = """Answer the question according to the image using a single word or phrase.
+{question}
+The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the question."""  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='docvqa',
+        pretty_name='DocVQA',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'DocVQA (Document Visual Question Answering) is a benchmark designed to evaluate AI systems on their ability to answer questions based on the content of document images, such as scanned pages, forms, or invoices. Unlike general visual question answering, it requires understanding not just the text extracted by OCR, but also the complex layout, structure, and visual elements of a document.',  # noqa: E501
+        dataset_id='lmms-lab/DocVQA',
+        subset_list=['DocVQA'],
+        metric_list=['anls'],
+        eval_split='validation',
+        prompt_template=PROMPT,
+    )
+)
+class DocVQAAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_aggregation_name = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+
+        input_text = PROMPT.format(question=record['question'])
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=json.dumps(record.get('answers')),  # answers is a list
+            metadata={
+                'questionId': record.get('questionId'),
+                'question_types': record.get('question_types'),
+                'docId': record.get('docId'),
+                'ucsf_document_id': record.get('ucsf_document_id'),
+                'ucsf_document_page_no': record.get('ucsf_document_page_no'),
+            }
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        import re
+
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return prediction.strip()

evalscope/benchmarks/drivelology/drivelology_binary_adapter.py

@@ -0,0 +1,170 @@
+# flake8: noqa: E501
+
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentText
+from evalscope.api.metric.scorer import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+DESCRIPTION = (
+    'Drivelology, a unique linguistic phenomenon characterised as "nonsense with depth" - '
+    'utterances that are syntactically coherent yet pragmatically paradoxical, emotionally loaded, '
+    'or rhetorically subversive.'
+)
+
+PROMPT_TEMPLATE = """
+#Instruction#:
+Classify whether the given text is a Drivelology sample or not.
+
+#Definition#:
+- Drivelology: Statements that appear logically coherent but contain deeper, often paradoxical meanings.
+These challenge conventional interpretation by blending surface-level nonsense with underlying depth,
+often incorporating elements of humor, irony, or sarcasm, and requiring contextual understanding and
+emotional insight to unravel their true significance.
+- non-Drivelology: This includes pure nonsense (grammatically correct but semantically meaningless
+statements, such as "Colourless green ideas sleep furiously") and normal sentences, including quotes
+or proverbs, that convey clear or straightforward information without the layered complexity
+characteristic of Drivelology.
+
+#Output Format#:
+You should try your best to answer "Yes" if the given input text is Drivelology, otherwise specify "No".
+The answer you give MUST be \"Yes\" or \"No\"".
+
+#Input Text#: {text}
+#Your Answer#:
+""".strip()  # noqa: E501
+
+FEWSHOT_PROMPT_TEMPLATE = """
+#Instruction#:
+Classify whether the given text is a Drivelology sample or not.
+
+#Definition#:
+- Drivelology: Statements that appear logically coherent but contain deeper, often paradoxical meanings.
+These challenge conventional interpretation by blending surface-level nonsense with underlying depth,
+often incorporating elements of humor, irony, or sarcasm, and requiring contextual understanding and
+emotional insight to unravel their true significance.
+- non-Drivelology: This includes pure nonsense (grammatically correct but semantically meaningless
+statements, such as "Colourless green ideas sleep furiously") and normal sentences, including quotes
+or proverbs, that convey clear or straightforward information without the layered complexity
+characteristic of Drivelology.
+
+#Output Format#:
+You should try your best to answer "Yes" if the given input text is Drivelology, otherwise specify "No".
+The answer you give MUST be \"Yes\" or \"No\"".
+
+Here are some examples of how to solve similar problems:
+
+#Input Text#: Saw a book called "how to solve 50 percent of your problems" so I bought 2 books.
+#Your Answer#: Yes
+
+#Input Text#: Colourless green ideas sleep furiously.
+#Your Answer#: No
+
+#Input Text#: I went to a restaurant, and saw this guy was choking. I gotta save him. And then I realized he was just speaking French.
+#Your Answer#: Yes
+
+#Input Text#: Either it is or it isn't.
+#Your Answer#: No
+
+#Input Text#: {text}
+#Your Answer#:
+""".strip()  # noqa: E501
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='drivel_binary',
+        pretty_name='DrivelologyBinaryClassification',
+        tags=[Tags.YES_NO],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/drivel-hub',
+        subset_list=['binary-classification'],
+        metric_list=['accuracy', 'precision', 'recall', 'f1_score', 'yes_ratio'],
+        aggregation='f1',
+        few_shot_num=0,
+        eval_split='test',
+        prompt_template='{question}',
+        few_shot_prompt_template='{question}'
+    )
+)
+class DrivelologyBinaryClassificationAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_overall_metric = False
+        if self.few_shot_num not in [0, 4]:
+            logger.warning(f'For DrivelologyBinaryClassification, use 4-shot by default.')
+            self.few_shot_num = 4
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        if self.few_shot_num > 0:
+            prompt = FEWSHOT_PROMPT_TEMPLATE.format(text=record['text'])
+        else:
+            prompt = PROMPT_TEMPLATE.format(text=record['text'])
+        content_list: List[Content] = [ContentText(text=prompt)]
+        answer = 'YES' if str(record['label']) == 'drivelology' else 'NO'  # 'YES' or 'NO'
+        return Sample(input=[ChatMessageUser(content=content_list)], target=answer, metadata={
+            'answer': answer,
+        })
+
+    def match_score(self, original_prediction, filtered_prediction, reference, task_state) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        # Check if the reference answer is in the filtered prediction
+        result = 1 if reference in filtered_prediction.strip().upper() else 0
+        score.value = {'acc': result}
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        """
+        Custom aggregation to compute accuracy, precision, recall, f1_score, and yes_ratio.
+        """
+
+        def compute_metrics(scores: List[SampleScore]):
+            tp = fp = tn = fn = 0
+            yes_count = 0
+            total_count = len(scores)
+
+            for ss in scores:
+                gt = ss.sample_metadata['answer'].strip().upper()
+                # Get prediction based on score
+                pred = gt if ss.score.main_value == 1 else ('NO' if gt == 'YES' else 'YES')
+                if pred == 'YES':
+                    yes_count += 1
+                if pred == 'YES' and gt == 'YES':
+                    tp += 1
+                elif pred == 'YES' and gt == 'NO':
+                    fp += 1
+                elif pred == 'NO' and gt == 'NO':
+                    tn += 1
+                elif pred == 'NO' and gt == 'YES':
+                    fn += 1
+
+            accuracy = (tp + tn) / total_count if total_count > 0 else 0.0
+            precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0
+            recall = tp / (tp + fn) if (tp + fn) > 0 else 0.0
+            f1_score = (2 * precision * recall) / (precision + recall) if (precision + recall) > 0 else 0.0
+            yes_ratio = yes_count / total_count if total_count > 0 else 0.0
+
+            return {
+                'accuracy': accuracy,
+                'precision': precision,
+                'recall': recall,
+                'f1_score': f1_score,
+                'yes_ratio': yes_ratio
+            }
+
+        overall_metrics = compute_metrics(sample_scores)
+        agg_scores = []
+        for metric_name, value in overall_metrics.items():
+            agg_scores.append(AggScore(metric_name=metric_name, score=value, num=len(sample_scores), metadata={}))
+
+        return agg_scores