evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/data_collection/data_collection_adapter.py

@@ -0,0 +1,215 @@
+import copy
+import os
+from collections import defaultdict
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DataAdapter, DefaultDataAdapter
+from evalscope.api.dataset import DatasetDict, LocalDataLoader, Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric.scorer import AggScore, SampleScore
+from evalscope.api.registry import get_benchmark, register_benchmark
+from evalscope.config import TaskConfig
+from evalscope.constants import DataCollection, Tags
+from evalscope.report.generator import ReportGenerator
+from evalscope.report.report import Report
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name=DataCollection.NAME,
+        pretty_name='Data-Collection',
+        dataset_id='',  # dataset_id need to be set
+        description='Custom Data collection, mixing multiple evaluation datasets for '
+        'a unified evaluation, aiming to use less data to achieve a more comprehensive '
+        'assessment of the model\'s capabilities. '
+        '[Usage Reference](https://evalscope.readthedocs.io/en/latest/advanced_guides/collection/index.html)',
+        tags=[Tags.CUSTOM],
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template='',
+    )
+)
+class DataCollectionAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        """
+        Data adapter for collection dataset.
+        """
+        super().__init__(**kwargs)
+
+    def load(self):
+        # Try to load dataset from local disk
+        dataset_name_or_path = self.dataset_id
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
+
+            # Load dataset from remote
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(dataset_name_or_path, allow_file_pattern='*.jsonl')
+
+        dataset = LocalDataLoader(
+            data_id_or_path=dataset_path,
+            split=self.eval_split,
+            sample_fields=self.record_to_sample,
+            subset='test',  # NOTE: using hardcoded test subset
+            limit=self.limit,
+            repeats=self.repeats,
+            shuffle=self.shuffle,
+        ).load()
+
+        test_dataset = DatasetDict({self.default_subset: dataset})
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object. Every record is a DatasetEntry.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        from evalscope.collections import DatasetEntry
+
+        entry = DatasetEntry.model_validate(record)
+        sample = Sample.model_validate(entry.prompt)
+
+        record_without_prompt = copy.deepcopy(record)
+        del record_without_prompt['prompt']
+        sample.metadata[DataCollection.INFO] = record_without_prompt  # keep all metadata
+        return sample
+
+    def _post_process_samples(self):
+        """Post process of each sample"""
+        self._initialize_adapters()
+
+    def _initialize_adapters(self):
+        """Init adapters for each dataset and create dataset id map"""
+        self.dataset_adapters: Dict[str, DataAdapter] = {}
+        self.dataset_name_map = defaultdict(lambda: defaultdict(list))
+
+        # load dataset args
+        dataset_args = copy.deepcopy(self._task_config.dataset_args)
+
+        # Iterate through each sample in the dataset
+        dataset = self.test_dataset[self.default_subset]
+        for sample in dataset:
+            collection_info = sample.metadata.get(DataCollection.INFO, {})
+            dataset_name = collection_info.get('dataset_name', '')
+            subset_name = collection_info.get('subset_name', '')
+            # create id mapping
+            self.dataset_name_map[dataset_name][subset_name].append(sample.id)
+
+            # update dataset args
+            cur_dataset_args = dataset_args.get(dataset_name, {})
+
+            # Initialize dataset adapter
+            if dataset_name not in self.dataset_adapters:
+                config = TaskConfig(dataset_args={dataset_name: cur_dataset_args})
+                self.dataset_adapters[dataset_name] = get_benchmark(dataset_name, config=config)
+
+    def _get_adapter(self, metadata: Dict[str, Any]) -> DataAdapter:
+        collection_info = metadata.get(DataCollection.INFO, {})
+        dataset_name = collection_info.get('dataset_name', '')
+        return self.dataset_adapters.get(dataset_name)
+
+    def run_inference(self, model, sample, output_dir, **kwargs) -> TaskState:
+        data_adapter = self._get_adapter(sample.metadata)
+        if not data_adapter:
+            raise ValueError(f'No data adapter found for sample: {sample}')
+
+        return data_adapter.run_inference(model, sample, output_dir, **kwargs)
+
+    def calculate_metrics(self, task_state) -> SampleScore:
+        data_adapter = self._get_adapter(task_state.metadata)
+        if not data_adapter:
+            raise ValueError(f'No data adapter found for task state: {task_state}')
+
+        return data_adapter.calculate_metrics(task_state)
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]):
+        import pandas as pd
+        from tabulate import tabulate
+
+        data = []
+        for sample_score in sample_scores:
+            collection_info = sample_score.sample_metadata[DataCollection.INFO]
+            main_score = sample_score.score.main_value
+            main_metric = sample_score.score.main_score_name
+
+            # use main score
+            data.append(
+                dict(
+                    task_type=collection_info['task_type'],
+                    categories=tuple(collection_info['categories']),
+                    dataset_name=collection_info['dataset_name'],
+                    subset_name=collection_info['subset_name'],
+                    tags=collection_info['tags'],
+                    sample_id=sample_score.sample_id,
+                    metric=main_metric,
+                    score=main_score
+                )
+            )
+
+        df = pd.DataFrame(data)
+
+        def aggregate_and_sort(df, group_by_cols):
+            # aggregate by group_by_cols, and calculate average_score and count
+            report_df = df.groupby(group_by_cols) \
+                .agg(average_score=('score', 'mean'), count=('score', 'size')) \
+                .reset_index()
+            report_df['average_score'] = report_df['average_score'].round(4)
+            report_df = report_df.sort_values(by='count', ascending=False) \
+                .to_dict(orient='records')
+            return report_df
+
+        # multi-level aggregation
+        subset_report_df = aggregate_and_sort(df, ['task_type', 'metric', 'dataset_name', 'subset_name'])
+        dataset_report_df = aggregate_and_sort(df, ['task_type', 'metric', 'dataset_name'])
+        task_report_df = aggregate_and_sort(df, ['task_type', 'metric'])
+
+        # explode tags to multiple rows
+        df_exploded_tags = df.explode('tags')
+        tag_report_df = aggregate_and_sort(df_exploded_tags, ['tags', 'metric'])
+
+        # process multi-level categories
+        df_categories = df.copy()
+        # multi-level aggregation for categories
+        max_depth = df_categories['categories'].apply(len).max()
+        for level in range(max_depth):
+            df_categories[f'category{level}'] = df_categories['categories'].apply(
+                lambda x: x[level] if len(x) > level else ''
+            )
+        category_report_df = aggregate_and_sort(
+            df_categories, [f'category{level}' for level in range(max_depth)] + ['metric']
+        )
+
+        # convert to dict format
+        report_dict = {
+            'subset_level': subset_report_df,
+            'dataset_level': dataset_report_df,
+            'task_level': task_report_df,
+            'tag_level': tag_report_df,
+            'category_level': category_report_df,
+        }
+
+        # record report
+        for level, data in report_dict.items():
+            table = tabulate(data, headers='keys', tablefmt='pretty', showindex=False)
+            logger.info(f'{level} Report:\n{table}')
+
+        return df
+
+    def generate_report(self, scores, model_name, output_dir, **kwargs) -> Report:
+        df = scores[self.default_subset]
+        report = ReportGenerator.gen_collection_report(df, self.name, model_name)
+        return report

evalscope/benchmarks/docmath/docmath_adapter.py

@@ -0,0 +1,143 @@
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+TEMPLATE_0SHOT = """Please read the following text and answer the question below.
+
+<text>
+{context}
+</text>
+
+{question}
+
+Format your response as follows: "Therefore, the answer is (insert answer here)"."""
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='docmath',
+        pretty_name='DocMath',
+        tags=[Tags.REASONING, Tags.MATH, Tags.LONG_CONTEXT],
+        description=
+        'DocMath-Eval is a comprehensive benchmark focused on numerical reasoning within specialized domains. It requires the model to comprehend long and specialized documents and perform numerical reasoning to answer the given question.',  # noqa: E501
+        dataset_id='yale-nlp/DocMath-Eval',
+        metric_list=['acc'],
+        subset_list=['complong_testmini', 'compshort_testmini', 'simplong_testmini', 'simpshort_testmini'],
+        eval_split='test',
+        prompt_template=TEMPLATE_0SHOT,
+    )
+)
+class DocMathAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._use_llm_judge = True  # Enable LLM judge for DocMath
+        self.split_as_subset = True  # Use split as subset for DocMath
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        ground_truth = record['ground_truth']
+
+        return Sample(
+            input=record['question'],
+            target=str(ground_truth),
+            metadata={
+                'question_id': record.get('question_id', ''),
+                'paragraphs': record['paragraphs'],
+                'answer_type': type(ground_truth).__name__
+            }
+        )
+
+    def format_prompt_template(self, sample):
+        context = '\n'.join(sample.metadata['paragraphs'])
+        question = sample.input
+        return self.prompt_template.format(context=context, question=question)
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
+        """
+        Extract the answer from the model prediction.
+        """
+        from .utils import extract_answer
+
+        extracted_answer = extract_answer(prediction)
+        return extracted_answer
+
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Calculate accuracy score by matching prediction with reference.
+        """
+        from .utils import get_acc
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        answer_type = task_state.metadata.get('answer_type', 'unknown')
+        accuracy = get_acc(prediction=filtered_prediction, gt=reference, answer_type=answer_type)
+        score.value = {'acc': accuracy}
+        score.main_score_name = 'acc'
+
+        return score
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Use LLM judge to evaluate the prediction against the reference.
+        """
+        from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.metadata.get('question', '')
+
+        # Get grading response
+        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=reference, answer_2=filtered_prediction)
+        orm_response = self.llm_judge.judge(prompt, system_prompt=GENERAL_ORM_PROMPT)
+
+        # Parse grading response
+        if 'YES' in orm_response:
+            accuracy = 1.0
+        else:
+            accuracy = 0.0
+
+        score.value = {'acc': accuracy}
+        score.explanation = f'LLM judge: {orm_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'acc'
+
+        return score

evalscope/benchmarks/docmath/utils.py

@@ -0,0 +1,219 @@
+import math
+import numpy as np
+import re
+from sympy import Rational
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+GENERAL_ORM_PROMPT = """You are an expert in verifying if two answers are the same.
+Your input is a problem and two answers, Answer 1 and Answer 2. You need to check if they are equivalent.
+Your task is to determine if two answers are equivalent, without attempting to solve the original problem.
+Compare the answers to verify they represent identical values or meaning, even when written in different forms or notations.
+
+Your output must follow the following format:
+1) Provide an explanation for why the answers are equivalent or not.
+2) Then provide your final answer in the form of: [[YES]] or [[NO]]
+"""  # noqa: E501
+
+ORM_USER_TEMPLATE = """
+Problem: {problem}
+Answer 1: {answer_1}
+Answer 2: {answer_2}
+"""
+
+
+def round_up_to_decimal(number, decimals):
+    factor = 10**decimals
+    return math.ceil(number * factor) / factor
+
+
+def is_number(string):
+    pattern = r'^[-+]?(\d{1,3}(,\d{3})*|(\d+))(\.\d+)?$'
+    match = re.match(pattern, string)
+    return bool(match)
+
+
+def is_scientific_number(string):
+    pattern = r'^[-+]?\d+(\.\d+)?e[-]?\d+$'
+    match = re.match(pattern, string)
+    return bool(match)
+
+
+def normalize(prediction: str):
+    # Preprocessing the string [Stage 1]
+    prediction = prediction.strip()
+    prediction = prediction.rstrip('.')
+    if not isinstance(prediction, str):
+        prediction = str(prediction) if prediction is not None else '0'
+
+    for money in ['£', '€', '¥', 'million', 'billion', 'thousand', 'US', 'USD', 'RMB']:
+        prediction = prediction.replace(money, '')
+
+    # Replace special tokens
+    if '=' in prediction:
+        prediction = prediction.split('=')[-1].strip()
+    if '≈' in prediction:
+        prediction = prediction.split('≈')[-1].strip()
+    if '`' in prediction:
+        prediction = prediction.replace('`', '')
+    if '%' in prediction:
+        prediction = prediction.replace('%', '')
+    if '$' in prediction:
+        prediction = prediction.replace('$', '')
+    if '°' in prediction:
+        prediction = prediction.replace('°', '')
+
+    # Detect the boolean keyword in the generation
+    if prediction in ['true', 'yes', 'false', 'no']:
+        if prediction == 'true' or prediction == 'yes':
+            prediction = 'True'
+        else:
+            prediction = 'False'
+    if 'True' in prediction or 'False' in prediction:
+        prediction = 'True' if 'True' in prediction else 'False'
+
+    # Detect the approximation keyword
+    if 'approximately' in prediction:
+        prediction = prediction.replace('approximately', '').strip()
+    if ' or ' in prediction:
+        prediction = prediction.split(' or ')[0]
+
+    # Drop the units before and after the number
+    if re.match(r'[-+]?(?:[\d,]*\.*\d+) [^0-9 ]+$', prediction):
+        prediction = re.search(r'([-+]?(?:[\d,]*\.*\d+)) [^0-9 ]+$', prediction).group(1)
+    if re.match(r'[^0-9 ]+ [-+]?(?:[\d,]*\.*\d+)$', prediction):
+        prediction = re.search(r'[^0-9 ]+ ([-+]?(?:[\d,]*\.*\d+))$', prediction).group(1)
+    if re.match(r'[-+]?(?:[\d,]*\.*\d+)[^\d]{1,2}$', prediction):
+        prediction = re.search(r'([-+]?(?:[\d,]*\.*\d+))[^\d]{1,2}$', prediction).group(1)
+    if re.match(r'[^-+\d]{1,2}(?:[\d,]*\.*\d+)$', prediction):
+        prediction = re.search(r'[^-+\d]{1,2}((?:[\d,]*\.*\d+))$', prediction).group(1)
+
+    # Preprocessing the number [Stage 1]
+    if '10^' in prediction:
+        prediction = re.sub(r'10\^(-?\d+)', r'math.pow(10, \1)', prediction)
+    if ' x ' in prediction:
+        prediction = prediction.replace(' x ', '*')
+    if ' × ' in prediction:
+        prediction = prediction.replace(' × ', '*')
+    if is_number(prediction):
+        prediction = prediction.replace(',', '')
+
+    # Preprocessing the option [Stage 3]
+    if '(a)' in prediction or '(b)' in prediction or '(c)' in prediction or '(d)' in prediction:
+        prediction = '"' + re.search(r'\([a-d]\)', prediction).group(0) + '"'
+
+    # If the prediction is empty, use dummy '0'
+    if not prediction:
+        prediction = '0'
+
+    # Converting the string answer to a number/list/bool/option
+    try:
+        prediction = eval(prediction)
+    except Exception:
+        # TO CHECK
+        prediction = 0
+
+    # Performing common type conversion
+    if isinstance(prediction, (set, tuple)):
+        prediction = list(prediction)
+        if isinstance(prediction[0], complex):
+            prediction = [tmp.real for tmp in prediction]
+        elif isinstance(prediction[0], Rational):
+            prediction = [float(tmp) for tmp in prediction]
+    elif isinstance(prediction, np.ndarray):
+        prediction = prediction.tolist()
+    else:
+        if isinstance(prediction, complex):
+            prediction = prediction.real
+        elif isinstance(prediction, Rational):
+            prediction = float(prediction)
+
+    return prediction
+
+
+def extract_answer(response: str):
+    """Parses the final answer from the model's response text.
+
+    Args:
+        response: Text extracted from the model's response
+
+    Returns:
+        The final answer as a numeric value (string), or None if not found
+    """
+    # Remove any asterisks or other unwanted characters
+    response = response.replace('*', '')
+    response = response.replace('(', '')
+    response = response.replace(')', '')
+
+    # Search for the pattern 'the answer is {final answer}.'
+    match = re.search(r'the answer is (\=?\≈?\`?\%?\$?\°?\£?\€?\¥?-?[0-9\.,]+)', response, re.IGNORECASE)
+
+    if match:
+        # Remove commas from the matched number (if any)
+        res = match.group(1).replace(',', '').rstrip('.')
+        return res
+    else:
+        return response
+
+
+def within_eps(pred: float, gt: float):
+    eps = abs(gt) * 0.0015
+    if pred >= gt - eps and pred <= gt + eps:
+        return True
+    else:
+        return False
+
+
+def compare_two_numbers(p, gt):
+    if isinstance(p, int) or isinstance(p, float):
+        pass
+    elif isinstance(p, list) or isinstance(p, bool) or isinstance(p, str):
+        return False
+    elif isinstance(p, tuple) or isinstance(p, complex) or isinstance(p, dict):
+        return False
+    else:
+        raise ValueError(p)
+
+    v1, v2 = max(abs(gt), abs(p)), min(abs(gt), abs(p))
+    if (v1 != 0 and v2 != 0) and int(math.log10(v1 / v2)) == math.log10(v1 / v2):
+        return True
+
+    if v2 <= v1 / 50 and within_eps(pred=v2 * 100, gt=v1):
+        return True
+    elif v2 <= v1 / 500 and within_eps(pred=v2 * 1000, gt=v1):
+        return True
+    elif v2 <= v1 / 50000 and within_eps(pred=v2 * 100000, gt=v1):
+        return True
+
+    if round_up_to_decimal(v1, 2) == round_up_to_decimal(v2, 2):
+        return True
+
+    return within_eps(pred=p, gt=gt)
+
+
+def get_acc(prediction, gt, answer_type, cot=True):
+    try:
+        if cot:
+            prediction = normalize(prediction)
+        else:
+            prediction = float(prediction)
+
+        assert answer_type in ['int', 'float', 'float64', 'bool'], answer_type
+        if isinstance(prediction, (str, int, float, bool)) or isinstance(prediction, list):
+            # Comparing prediction against the reference
+            if answer_type in ['bool']:
+                acc = int(prediction == bool(gt))
+            elif answer_type == 'int':
+                acc = int(compare_two_numbers(prediction, int(gt)))
+            elif answer_type == 'float' or answer_type == 'float64':
+                acc = int(compare_two_numbers(prediction, float(gt)))
+            else:
+                acc = 0
+        else:
+            acc = 0
+            logger.error('Error: ', prediction, type(prediction))
+        return acc
+    except Exception:
+        return 0

evalscope/benchmarks/docvqa/docvqa_adapter.py

@@ -0,0 +1,67 @@
+import json
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+PROMPT = """Answer the question according to the image using a single word or phrase.
+{question}
+The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the question."""  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='docvqa',
+        pretty_name='DocVQA',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'DocVQA (Document Visual Question Answering) is a benchmark designed to evaluate AI systems on their ability to answer questions based on the content of document images, such as scanned pages, forms, or invoices. Unlike general visual question answering, it requires understanding not just the text extracted by OCR, but also the complex layout, structure, and visual elements of a document.',  # noqa: E501
+        dataset_id='lmms-lab/DocVQA',
+        subset_list=['DocVQA'],
+        metric_list=['anls'],
+        eval_split='validation',
+        prompt_template=PROMPT,
+    )
+)
+class DocVQAAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_aggregation_name = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+
+        input_text = PROMPT.format(question=record['question'])
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=json.dumps(record.get('answers')),  # answers is a list
+            metadata={
+                'questionId': record.get('questionId'),
+                'question_types': record.get('question_types'),
+                'docId': record.get('docId'),
+                'ucsf_document_id': record.get('ucsf_document_id'),
+                'ucsf_document_page_no': record.get('ucsf_document_page_no'),
+            }
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        import re
+
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return prediction.strip()