evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/drop/drop_adapter.py

@@ -0,0 +1,155 @@
+import ast
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+DROP_EXAMPLES = '''Some examples of passages and Q&A are provided below.
+
+# Examples
+---
+Passage: Trunajaya rebellion  or Trunajaya War was the ultimately unsuccessful rebellion waged by the Madurese prince Trunajaya and fighters from Makassar against the Mataram Sultanate and its Dutch East India Company  supporters in Java  during the 1670s. The rebellion was initially successful: the rebels defeated the royal army at Gegodog , captured most of the Javanese north coast, and took the Mataram capital Plered . King Amangkurat I died during the retreat of the royal court. His son and successor, Amangkurat II, requested help from the VOC in exchange for financial remuneration and geopolitical concessions. The VOC\'s subsequent involvement turned the tide of the war. VOC and Mataram forces recovered lost territories and overran Trunajaya\'s new capital at Kediri . However, the rebellion continued until the capture of Trunajaya at the end of 1679, and the defeat, death, or surrender of the other rebel leaders . Trunajaya was killed by Amangkurat II personally in 1680 while a prisoner of the VOC. After his father\'s death in 1677, Amangkurat II also faced rival claims to the throne. The most serious rival was his brother Pangeran Puger, who took the capital Plered in 1677 and did not surrender until 1681.
+Question: How many years was it between Trunajaya\'s capture and his death while prisoner of the VOC?
+Answer:  1
+
+---
+Passage: Led by former Giant Kurt Warner, the defending NFC champions took the field at Giants Stadium against a Giants team still reeling from their bad loss in New Orleans. The Giants scored first, sending Jacobs in for a 4-yard touchdown run following a Terrell Thomas interception. Later, Arizona running back Beanie Wells scored his first career touchdown on a 13-yard rush. Manning responded by throwing a 62-yard touchdown to Nicks for his longest reception of the year. In the second half, the Cardinals\' Tim Hightower and Jason Wright scored touchdowns. But it was turnovers that decided this game; Manning\'s 3 interceptions were as many as he had thrown all season. The Giants scored only 3 points in the second half, ending the game on an interception to Antrel Rolle. The Giants notable streak of 38 consecutive starts by the same offensive line unit was ended here, as offensive tackle Kareem McKenzie missed the game with a groin injury. McKenzie returned the following week.
+Question: Which player made the first score of the game?
+Answer:  Jacobs
+
+---
+Passage: Hoping to rebound from their road loss to the Bills, the Chargers flew to Wembley Stadium for the 2008 International Series game with the New Orleans Saints. In the first quarter, San Diego trailed early as kicker Taylor Mehlhaff got a 23-yard field goal.  The \'Bolts would respond with kicker Nate Kaeding getting a 33-yard field goal.  In the second quarter, New Orleans regained the lead as QB Drew Brees (a former Charger) completed a 12-yard TD pass to WR Devery Henderson (with a failed PAT) and RB Deuce McAllister getting a 1-yard TD run.  San Diego answered as QB Philip Rivers completed a 12-yard TD pass to RB LaDainian Tomlinson, but the Saints replied with Brees completing a 30-yard TD pass to WR Lance Moore.  The Chargers closed out the half with Rivers completing a 12-yard TD pass to TE Antonio Gates. In the third quarter, New Orleans increased its lead Brees completing a 1-yard TD pass to TE Mark Campbell, after a very controversial Pass interference call on cornerback Cletis Gordon put the Saints on the 1-yard line.  The \'Bolts would answer with Kaeding getting a 24-yard field goal.  In the fourth quarter, the Saints continued to build its lead as FB Mike Karney got a 1-yard TD run.  San Diego tried to rally as Kaeding nailed a 31-yard field goal, Rivers completed a 14-yard TD pass to WR Vincent Jackson, and Brees giving the \'Bolts a safety via an incomplete pass thrown into the back of his own endzone.  However, New Orleans\' defense stiffened for the win. With the loss, the Chargers went into their bye week at 3-5.
+Question: How many total yards of touchdown passes did Drew Brees make?
+Answer:  43
+
+'''  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='drop',
+        pretty_name='DROP',
+        tags=[Tags.REASONING],
+        description=
+        'The DROP (Discrete Reasoning Over Paragraphs) benchmark is designed to evaluate the reading comprehension and reasoning capabilities of AI models. It includes a variety of tasks that require models to read passages and answer questions based on the content.',  # noqa: E501
+        dataset_id='AI-ModelScope/DROP',
+        metric_list=['em', 'f1'],
+        few_shot_num=3,
+        train_split=None,
+        eval_split='validation',
+        prompt_template=
+        'You will be asked to read a passage and answer a question. {drop_examples}\n# Your Task\n\n---\n{query}\n\nThink step by step, then write a line of the form "Answer: $ANSWER" at the end of your response.',  # noqa: E501
+    )
+)
+class DROPAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        if self.few_shot_num != 0 and self.few_shot_num != 3:
+            self.few_shot_num = 3
+            logger.info(f'Few shot num is set to {self.few_shot_num} for DROP dataset by system.')
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        from .utils import _get_gold_answers
+
+        # Parse gold answers
+        gold_answers = _get_gold_answers(record)
+
+        return Sample(
+            input=record['question'],
+            target=str(gold_answers),
+            metadata={
+                'passage': record['passage'],
+                'answer': record['answer'],
+                'validated_answers': record['validated_answers']
+            }
+        )
+
+    def format_prompt_template(self, sample: Sample) -> str:
+        drop_examples = ''
+        query = f"Passage: {sample.metadata['passage']}\nQuestion: {sample.input}"
+
+        return self.prompt_template.format(
+            drop_examples=drop_examples,
+            query=query,
+        )
+
+    def format_fewshot_template(self, fewshot, sample):
+        drop_examples = DROP_EXAMPLES
+        query = f"Passage: {sample.metadata['passage']}\nQuestion: {sample.input}"
+
+        return self.prompt_template.format(
+            drop_examples=drop_examples,
+            query=query,
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
+        """
+        Extract the answer from the model prediction.
+        """
+        match = re.search(r'(?i)Answer\s*:\s*([^\n]+)', prediction)
+        extracted_answer = match.group(1) if match else prediction
+        return extracted_answer
+
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Calculate accuracy score by matching prediction with reference answers.
+        """
+        import numpy as np
+
+        from .utils import _align_bags, _answer_to_bags
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        max_em = 0
+        max_f1 = 0
+        reference = ast.literal_eval(reference) if isinstance(reference, str) else reference
+        for gold_answer in reference:
+            # Convert the answers to bags of answers
+            predicted_bags = _answer_to_bags(filtered_prediction)
+            gold_bags = _answer_to_bags(gold_answer)
+
+            if set(predicted_bags[0]) == set(gold_bags[0]) and len(predicted_bags[0]) == len(gold_bags[0]):
+                exact_match = 1.0
+            else:
+                exact_match = 0.0
+
+            f1_per_bag = _align_bags(predicted_bags[1], gold_bags[1])
+            f1_score = np.mean(f1_per_bag)
+            f1_score = round(f1_score, 2)
+            # Check if the answer is empty
+            if gold_answer[0].strip():
+                max_em = max(max_em, exact_match)
+                max_f1 = max(max_f1, f1_score)
+
+        score.value = {'em': max_em, 'f1': max_f1}
+        score.main_score_name = 'f1'
+
+        return score

evalscope/benchmarks/drop/utils.py

@@ -0,0 +1,156 @@
+import numpy as np
+import re
+import string
+from typing import List
+
+_ARTICLES = re.compile(r'\b(a|an|the)\b', re.UNICODE)
+
+
+def _answer_to_bags(answer):
+    if isinstance(answer, (list, tuple)):
+        raw_spans = answer
+    else:
+        raw_spans = [answer]
+    normalized_spans = []
+    token_bags = []
+    for raw_span in raw_spans:
+        normalized_span = _normalize(raw_span)
+        normalized_spans.append(normalized_span)
+        token_bags.append(set(normalized_span.split()))
+    return normalized_spans, token_bags
+
+
+def _is_number(text):
+    try:
+        float(text)
+        return True
+    except ValueError:
+        return False
+
+
+def _remove_articles(text):
+    return _ARTICLES.sub(' ', text)
+
+
+def _white_space_fix(text):
+    return ' '.join(text.split())
+
+
+def _remove_punc(text):
+    exclude = set(string.punctuation)
+    if not _is_number(text):
+        return ''.join(ch for ch in text if ch not in exclude)
+    else:
+        return text
+
+
+def _fix_number(text):
+    return str(float(text)) if _is_number(text) else text
+
+
+def _tokenize(text):
+    return re.split(' |-', text)
+
+
+def _normalize(answer):
+    tokens = [
+        _white_space_fix(_remove_articles(_fix_number(_remove_punc(token.lower())))) for token in _tokenize(answer)
+    ]
+    tokens = [token for token in tokens if token.strip()]
+    normalized = ' '.join(tokens).strip()
+    return normalized
+
+
+def _compute_f1(predicted_bag, gold_bag):
+    intersection = len(gold_bag.intersection(predicted_bag))
+    if not predicted_bag:
+        precision = 1.0
+    else:
+        precision = intersection / float(len(predicted_bag))
+    if not gold_bag:
+        recall = 1.0
+    else:
+        recall = intersection / float(len(gold_bag))
+    f1 = ((2 * precision * recall) / (precision + recall) if not (precision == 0.0 and recall == 0.0) else 0.0)
+    return f1
+
+
+def _match_numbers_if_present(gold_bag, predicted_bag):
+    gold_numbers = {word for word in gold_bag if _is_number(word)}
+    predicted_numbers = {word for word in predicted_bag if _is_number(word)}
+    if (not gold_numbers) or gold_numbers.intersection(predicted_numbers):
+        return True
+    return False
+
+
+def _align_bags(predicted, gold):
+    """
+    Takes gold and predicted answer sets and first finds the optimal 1-1 alignment
+    between them and gets maximum metric values over all the answers.
+    """
+    from scipy.optimize import linear_sum_assignment
+
+    scores = np.zeros([len(gold), len(predicted)])
+    for gold_index, gold_item in enumerate(gold):
+        for pred_index, pred_item in enumerate(predicted):
+            if _match_numbers_if_present(gold_item, pred_item):
+                scores[gold_index, pred_index] = _compute_f1(pred_item, gold_item)
+    row_ind, col_ind = linear_sum_assignment(-scores)
+
+    max_scores = np.zeros([max(len(gold), len(predicted))])
+    for row, column in zip(row_ind, col_ind):
+        max_scores[row] = max(max_scores[row], scores[row, column])
+    return max_scores
+
+
+def parse_answer(answer):
+    # NOTE: Everything is returned as a tuple for uniformity and hashability.
+    if answer['number'] != '':
+        return (str(answer['number']), )
+    if answer['spans'] != []:
+        return tuple(answer['spans'])
+    return (' '.join([answer['date']['day'], answer['date']['month'], answer['date']['year']]).strip(), )
+
+
+def _get_gold_answers(input_d: dict) -> List[str]:
+    """
+    Parse the raw input labels (gold).
+    """
+
+    def _flatten_validated_answers(validated_answers: dict) -> List[dict]:
+        """
+        Flatten the validated_answers structure into a list of answer dictionaries.
+
+        Expected input:
+            validated_answers: {
+                'number': [...],
+                'date':   [...],
+                'spans':  [...]
+            }
+
+        Each returned dict has keys: 'number', 'date', 'spans'.
+        If the input lists have different lengths, iteration stops at the shortest.
+        """
+        # Safely read lists from the input dict (default to empty lists)
+        numbers = validated_answers.get('number', [])
+        dates = validated_answers.get('date', [])
+        spans = validated_answers.get('spans', [])
+
+        # Ensure we only iterate as far as the shortest sequence to avoid IndexError
+        length = min(len(numbers), len(dates), len(spans))
+
+        flattened: List[dict] = []
+        for num, date, sp in zip(numbers[:length], dates[:length], spans[:length]):
+            flattened.append({'number': num, 'date': date, 'spans': sp})
+        return flattened
+
+    answers = []
+    answers_set = set()
+    candidates = [input_d['answer']] + _flatten_validated_answers(input_d['validated_answers'])
+    for candidate in candidates:
+        answer = parse_answer(candidate)
+        if answer in answers_set:
+            continue
+        answers_set.add(answer)
+        answers.append(answer)
+    return answers

evalscope/benchmarks/frames/frames_adapter.py

@@ -0,0 +1,175 @@
+import os
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import DatasetDict, LocalDataLoader, Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+TEMPLATE_0SHOT = """Please read the following text and answer the question below.
+
+<text>
+{context}
+</text>
+
+{question}
+
+Format your response as follows: "Therefore, the answer is (insert answer here)"."""
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='frames',
+        pretty_name='FRAMES',
+        tags=[Tags.REASONING, Tags.LONG_CONTEXT],
+        description=
+        'FRAMES is a comprehensive evaluation dataset designed to test the capabilities of Retrieval-Augmented Generation (RAG) systems across factuality, retrieval accuracy, and reasoning.',  # noqa: E501
+        dataset_id='iic/frames',
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=TEMPLATE_0SHOT,
+    )
+)
+class FramesAdapter(DefaultDataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self._use_llm_judge = True  # Enable LLM judge for FRAMES
+
+    def load(self):
+        # Try to load dataset from local disk
+        dataset_name_or_path = self.dataset_id
+        if os.path.exists(dataset_name_or_path):
+            logger.info(f'Loading dataset from {dataset_name_or_path}')
+            dataset_path = dataset_name_or_path
+        else:
+            from modelscope import dataset_snapshot_download
+
+            # Load dataset from remote
+            logger.info(f'Loading dataset from modelscope: > dataset_name: {dataset_name_or_path}')
+            # download dataset snapshot
+            dataset_path = dataset_snapshot_download(dataset_name_or_path, allow_file_pattern='test.jsonl')
+
+        dataset = LocalDataLoader(
+            data_id_or_path=dataset_path,
+            split=self.eval_split,
+            sample_fields=self.record_to_sample,
+            subset='test',
+            limit=self.limit,
+            repeats=self.repeats,
+            shuffle=self.shuffle,
+        ).load()
+
+        test_dataset = DatasetDict({'test': dataset})
+
+        return test_dataset, None
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        context = '\n'.join([f"{i['title']}\n{i['text']}" for i in record['wiki_items']])
+        question = record['Prompt']
+
+        return Sample(
+            input=question, target=record['Answer'], metadata={
+                'context': context,
+                'wiki_items': record['wiki_items']
+            }
+        )
+
+    def format_prompt_template(self, sample):
+        context = sample.metadata['context']
+        question = sample.input
+        return self.prompt_template.format(context=context, question=question)
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
+        """
+        Extract the answer from the model prediction.
+        """
+        response = prediction.replace('*', '')
+
+        if 'the answer is' in response:
+            ans = response.rsplit('the answer is', 1)[-1].strip().strip('.').strip()
+        else:
+            ans = ''
+
+        return ans
+
+    def match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Calculate accuracy score by matching prediction with reference.
+        """
+        from evalscope.metrics import exact_match
+        from .utils import normalize_answer
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        gold = normalize_answer(reference)
+        pred = normalize_answer(filtered_prediction)
+        accuracy = exact_match(gold=gold, pred=pred)
+
+        score.value = {'acc': accuracy}
+        score.main_score_name = 'acc'
+
+        return score
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        """
+        Use LLM judge to evaluate the prediction against the reference.
+        """
+        from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # Get grading response
+        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=reference, answer_2=filtered_prediction)
+        orm_response = self.llm_judge.judge(prompt, system_prompt=GENERAL_ORM_PROMPT)
+
+        # Parse grading response
+        if 'YES' in orm_response:
+            accuracy = 1.0
+        else:
+            accuracy = 0.0
+
+        score.value = {'acc': accuracy}
+        score.explanation = f'LLM judge: {orm_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id
+        }
+        score.main_score_name = 'acc'
+
+        return score

evalscope/benchmarks/frames/utils.py

@@ -0,0 +1,37 @@
+import re
+import string
+
+
+def normalize_answer(s):
+
+    def remove_articles(text):
+        return re.sub(r'\b(a|an|the)\b', ' ', text)
+
+    def white_space_fix(text):
+        return ' '.join(text.split())
+
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return ''.join(ch for ch in text if ch not in exclude)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+GENERAL_ORM_PROMPT = """You are an expert in verifying if two answers are the same.
+Your input is a problem and two answers, Answer 1 and Answer 2. You need to check if they are equivalent.
+Your task is to determine if two answers are equivalent, without attempting to solve the original problem.
+Compare the answers to verify they represent identical values or meaning, even when written in different forms or notations.
+
+Your output must follow the following format:
+1) Provide an explanation for why the answers are equivalent or not.
+2) Then provide your final answer in the form of: [[YES]] or [[NO]]
+"""  # noqa: E501
+
+ORM_USER_TEMPLATE = """
+Problem: {problem}
+Answer 1: {answer_1}
+Answer 2: {answer_2}
+"""