evalscope 0.15.1__py3-none-any.whl → 0.16.1__py3-none-any.whl

evalscope/benchmarks/drop/utils.py

@@ -0,0 +1,59 @@
+import re
+import string
+
+_ARTICLES = re.compile(r'\b(a|an|the)\b', re.UNICODE)
+
+
+def _answer_to_bags(answer):
+    if isinstance(answer, (list, tuple)):
+        raw_spans = answer
+    else:
+        raw_spans = [answer]
+    normalized_spans = []
+    token_bags = []
+    for raw_span in raw_spans:
+        normalized_span = _normalize(raw_span)
+        normalized_spans.append(normalized_span)
+        token_bags.append(set(normalized_span.split()))
+    return normalized_spans, token_bags
+
+
+def _is_number(text):
+    try:
+        float(text)
+        return True
+    except ValueError:
+        return False
+
+
+def _remove_articles(text):
+    return _ARTICLES.sub(' ', text)
+
+
+def _white_space_fix(text):
+    return ' '.join(text.split())
+
+
+def _remove_punc(text):
+    exclude = set(string.punctuation)
+    if not _is_number(text):
+        return ''.join(ch for ch in text if ch not in exclude)
+    else:
+        return text
+
+
+def _fix_number(text):
+    return str(float(text)) if _is_number(text) else text
+
+
+def _tokenize(text):
+    return re.split(' |-', text)
+
+
+def _normalize(answer):
+    tokens = [
+        _white_space_fix(_remove_articles(_fix_number(_remove_punc(token.lower())))) for token in _tokenize(answer)
+    ]
+    tokens = [token for token in tokens if token.strip()]
+    normalized = ' '.join(tokens).strip()
+    return normalized

evalscope/benchmarks/frames/frames_adapter.py

@@ -0,0 +1,90 @@
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import EvalType, OutputType
+from evalscope.metrics import LLMJudge, exact_match
+
+TEMPLATE_0SHOT = """Please read the following text and answer the question below.
+
+<text>
+{context}
+</text>
+
+{question}
+
+Format your response as follows: "Therefore, the answer is (insert answer here)"."""
+
+
+@Benchmark.register(
+    name='frames',
+    pretty_name='FRAMES',
+    description=
+    'FRAMES is a comprehensive evaluation dataset designed to test the capabilities of Retrieval-Augmented Generation (RAG) systems across factuality, retrieval accuracy, and reasoning.',  # noqa: E501
+    dataset_id='iic/frames',
+    model_adapter=OutputType.GENERATION,
+    output_types=[OutputType.GENERATION],
+    metric_list=['AverageAccuracy'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    prompt_template=TEMPLATE_0SHOT,
+)
+class FramesAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def load(self, **kwargs):
+        # default load with snapshot
+        kwargs['file_structure'] = {'default': ['test.jsonl']}
+        data_dict = super().load_with_snapshot(**kwargs)
+        return data_dict
+
+    def gen_prompt(self, input_d: dict, subset_name: str, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate model prompt from input data.
+        """
+        context = '\n'.join([f"{i['title']}\n{i['text']}" for i in input_d['wiki_items']])
+        question = input_d['Prompt']
+        prompt = self.prompt_template.format(context=context, question=question)
+        return self.gen_prompt_data(prompt)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return input_d['Answer']
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        response = result.replace('*', '')
+
+        if 'the answer is' in response:
+            ans = response.rsplit('the answer is', 1)[-1].strip().strip('.').strip()
+        else:
+            ans = ''
+
+        return ans
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        from .utils import normalize_answer
+        gold = normalize_answer(gold)
+        pred = normalize_answer(pred)
+        return exact_match(gold=gold, pred=pred)
+
+    def llm_match(self, gold: str, pred: str, judge: LLMJudge, **kwargs) -> float:
+        from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE
+
+        raw_input = kwargs.get('raw_input', None)
+        question = raw_input['Prompt']
+        # get grading response
+        prompt = ORM_USER_TEMPLATE.format(problem=question, answer_1=gold, answer_2=pred)
+        orm_response = judge(prompt=prompt, system_prompt=GENERAL_ORM_PROMPT)
+        # parse grading response
+        if 'YES' in orm_response:
+            return 1.0
+        else:
+            return 0.0

evalscope/benchmarks/frames/utils.py

@@ -0,0 +1,37 @@
+import re
+import string
+
+
+def normalize_answer(s):
+
+    def remove_articles(text):
+        return re.sub(r'\b(a|an|the)\b', ' ', text)
+
+    def white_space_fix(text):
+        return ' '.join(text.split())
+
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return ''.join(ch for ch in text if ch not in exclude)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+GENERAL_ORM_PROMPT = """You are an expert in verifying if two answers are the same.
+Your input is a problem and two answers, Answer 1 and Answer 2. You need to check if they are equivalent.
+Your task is to determine if two answers are equivalent, without attempting to solve the original problem.
+Compare the answers to verify they represent identical values or meaning, even when written in different forms or notations.
+
+Your output must follow the following format:
+1) Provide an explanation for why the answers are equivalent or not.
+2) Then provide your final answer in the form of: [[YES]] or [[NO]]
+"""  # noqa: E501
+
+ORM_USER_TEMPLATE = """
+Problem: {problem}
+Answer 1: {answer_1}
+Answer 2: {answer_2}
+"""

evalscope/benchmarks/general_qa/general_qa_adapter.py

@@ -4,7 +4,7 @@ from collections import defaultdict
 from typing import List, Optional, Union
 
 from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import bleu_ngram_one_sample, compute_rouge_score_one_sample_zh, mean
+from evalscope.metrics import mean
 from evalscope.utils.io_utils import jsonl_to_list
 from evalscope.utils.logger import get_logger
 
@@ -112,9 +112,13 @@ class GeneralQAAdapter(DataAdapter):
         """
         res = dict()
         if 'AverageRouge' in self.metric_list:
+            from evalscope.metrics.rouge_metric import compute_rouge_score_one_sample_zh
+
             rouge_dict = compute_rouge_score_one_sample_zh([pred], [gold])
             res.update(rouge_dict)
         if 'AverageBLEU' in self.metric_list:
+            from evalscope.metrics import bleu_ngram_one_sample
+
             bleu_dict = bleu_ngram_one_sample(pred, gold)
             res.update(bleu_dict)
         return res

evalscope/benchmarks/needle_haystack/needle_haystack_adapter.py

@@ -0,0 +1,341 @@
+from itertools import product
+from tqdm import tqdm
+from typing import TYPE_CHECKING, List, Union
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.constants import AnswerKeys, EvalType
+from evalscope.metrics import LLMJudge, exact_match
+from evalscope.metrics.metrics import mean
+from evalscope.utils import get_logger
+
+if TYPE_CHECKING:
+    from evalscope.report import Report
+
+logger = get_logger()
+
+PROMPT_TEMPLATE = """Please read the following text and answer the question below.
+
+<text>
+{context}
+</text>
+
+<question>
+{question}
+</question>
+
+Don't give information outside the document or repeat your findings."""
+
+
+@Benchmark.register(
+    name='needle_haystack',
+    pretty_name='Needle in a Haystack',
+    description='Needle in a Haystack is a benchmark focused on information retrieval tasks. \
+    It requires the model to find specific information within a large corpus of text.',
+    dataset_id='AI-ModelScope/Needle-in-a-Haystack-Corpus',
+    metric_list=['AverageAccuracy'],
+    subset_list=['english', 'chinese'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    system_prompt='You are a helpful AI bot that answers questions for a user. Keep your response short and direct',
+    prompt_template=PROMPT_TEMPLATE,
+    extra_params={
+        'retrieval_question': 'What is the best thing to do in San Francisco?',
+        'needles':
+        ['\nThe best thing to do in San Francisco is eat a sandwich and sit in Dolores Park on a sunny day.\n'],
+        'context_lengths_min': 1000,
+        'context_lengths_max': 32000,
+        'context_lengths_num_intervals': 10,
+        'document_depth_percent_min': 0,
+        'document_depth_percent_max': 100,
+        'document_depth_percent_intervals': 10,
+        'tokenizer_path': 'Qwen/Qwen3-0.6B',
+    })
+class NeedleHaystackAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.llm_as_a_judge = True
+        # set extra params
+        extra_params = kwargs.get('extra_params', {})
+        self.retrieval_question = extra_params.get('retrieval_question',
+                                                   'What is the best thing to do in San Francisco?')
+        self.needles = extra_params.get(
+            'needles',
+            ['\nThe best thing to do in San Francisco is eat a sandwich and sit in Dolores Park on a sunny day.\n'])
+        self.context_lengths_min = extra_params.get('context_lengths_min', 1000)
+        self.context_lengths_max = extra_params.get('context_lengths_max', 32000)
+        self.context_lengths_num_intervals = extra_params.get('context_lengths_num_intervals', 10)
+        self.document_depth_percent_min = extra_params.get('document_depth_percent_min', 0)
+        self.document_depth_percent_max = extra_params.get('document_depth_percent_max', 100)
+        self.document_depth_percent_intervals = extra_params.get('document_depth_percent_intervals', 10)
+        self.tokenizer_path = extra_params.get('tokenizer_path', 'Qwen/Qwen3-0.6B')
+
+        self.__init_tokenizer()
+        self.__init_length()
+
+    def __init_length(self):
+        """ Initialize context lengths and document depth percentages based on the provided parameters."""
+        import numpy as np
+
+        self.context_lengths = np.round(
+            np.linspace(
+                self.context_lengths_min,
+                self.context_lengths_max,
+                num=self.context_lengths_num_intervals,
+                endpoint=True)).astype(int)
+
+        self.document_depth_percents = np.round(
+            np.linspace(
+                self.document_depth_percent_min,
+                self.document_depth_percent_max,
+                num=self.document_depth_percent_intervals,
+                endpoint=True)).astype(int)
+
+    def __init_tokenizer(self):
+        """ Initialize the tokenizer based on the provided tokenizer path."""
+        from modelscope import AutoTokenizer
+        self.tokenizer = AutoTokenizer.from_pretrained(self.tokenizer_path)
+
+    def load(self, **kwargs):
+        # default load with snapshot
+        kwargs['file_structure'] = {'english': ['PaulGraham_Essays.txt'], 'chinese': ['Journey_to_the_West.txt']}
+        data_dict = super().load_with_snapshot(**kwargs)
+        return data_dict
+
+    def gen_prompts(self, data_dict: dict) -> dict:
+        """
+        Generate dataset prompts from raw input, unify the prompt format for different datasets.
+
+        Args:
+            data_dict: {'english': {'test': [sample_d_1, sample_d_2, ...]},
+                        'chinese': {'test': [sample_d_1, sample_d_2, ...]}}
+
+        Returns:
+            {'subset_name': [prompt_d_1, prompt_d_2, ...]}
+            prompt_d_i (dict): refer to the output of gen_prompt method.
+
+        e.g. train -- few-shot data, test -- target dataset to evaluate.
+        """
+        res_dict: dict = {}
+
+        for sub_name, sub_data_dict in data_dict.items():
+            res_dict[sub_name] = []
+            for sample_d in sub_data_dict[self.eval_split]:
+                # Generate prompts for each sample in the dataset
+                tokens_context = self._get_context_tokens(sample_d['text'])
+                for context_length, depth_percent in tqdm(
+                        product(self.context_lengths, self.document_depth_percents),
+                        desc=f'Generating {sub_name} prompts'):
+                    # Insert needles into the context at the specified depth percentage
+                    context = self._insert_needles(tokens_context, depth_percent, context_length)
+                    # Build the input dictionary for the prompt
+                    input_d = {
+                        'context_length': int(context_length),
+                        'depth_percent': int(depth_percent),
+                        'question': self.retrieval_question,
+                        'answer': '\n'.join(self.needles),
+                        'context': context,
+                    }
+                    prompt_d = self.gen_prompt(input_d=input_d)
+                    prompt_d[AnswerKeys.RAW_INPUT] = input_d
+                    res_dict[sub_name].append(prompt_d)
+
+        return res_dict
+
+    def _get_context_tokens(self, input_context: str) -> list:
+        """
+        Encodes the context string into tokens using the tokenizer, ensuring the tokenized context
+        is at least as long as the maximum context length required.
+
+        Args:
+            input_context (str): The context string to be tokenized.
+
+        Returns:
+            List[int]: A list of token IDs representing the context.
+        """
+        max_context_length = max(self.context_lengths)
+        context = input_context
+        tokens_context = self.tokenizer.encode(context, add_special_tokens=False)
+        # Repeat the context until reaching the required length
+        while len(tokens_context) < max_context_length:
+            context += '\n' + input_context
+            tokens_context = self.tokenizer.encode(context, add_special_tokens=False)
+        return tokens_context
+
+    def _insert_needles(self, tokens_context, depth_percent, context_length):
+        """
+        Inserts multiple needles (specific facts or pieces of information) into the original context string at
+        designated depth percentages, effectively distributing these needles throughout the context. This method
+        is designed to test a model's ability to retrieve specific information (needles) from a larger body of text
+        (haystack) based on the placement depth of these needles.
+
+        The method first encodes the context and each needle into tokens to calculate their lengths in tokens.
+        It then adjusts the context length to accommodate the final buffer length. This is crucial for ensuring
+        that the total token count (context plus needles) does not exceed the maximum allowable context length,
+        which might otherwise lead to information being truncated.
+
+        This approach calculates the initial insertion point for the first needle as before but then calculates even
+        spacing for the remaining needles based on the remaining context length. It ensures that needles are
+        distributed as evenly as possible throughout the context after the first insertion.
+
+        Args:
+            tokens_context (List[int]): The original context tokens.
+            depth_percent (float): The depth percent at which to insert the needles.
+            context_length (int): The total length of the context in tokens, adjusted for final buffer.
+
+        Returns:
+            str: The new context with needles inserted.
+        """
+
+        context_length -= 150
+
+        # Calculate the total length of all needles in tokens
+        total_needles_length = sum(len(self.tokenizer.encode(needle)) for needle in self.needles)
+
+        # Ensure context length accounts for needles
+        if len(tokens_context) + total_needles_length > context_length:
+            tokens_context = tokens_context[:context_length - total_needles_length]
+
+        # To evenly distribute the needles, we calculate the intervals they need to be inserted.
+        depth_percent_interval = (100 - depth_percent) / len(self.needles)
+
+        # Reset the insertion percentages list for the current context
+        self.insertion_percentages = []
+
+        # Insert needles at calculated points
+        for needle in self.needles:
+
+            tokens_needle = self.tokenizer.encode(needle)
+
+            if depth_percent == 100:
+                # If your depth percent is 100 (which means your needle is the last thing in the doc),
+                # throw it at the end
+                tokens_context = tokens_context + tokens_needle
+            else:
+                # Go get the position (in terms of tokens) to insert your needle
+                insertion_point = int(len(tokens_context) * (depth_percent / 100))
+
+                # tokens_new_context represents the tokens before the needle
+                tokens_new_context = tokens_context[:insertion_point]
+
+                # We want to make sure that we place our needle at a sentence break
+                # so we first see what token a '.' is
+                period_tokens = self.tokenizer.encode('.') + self.tokenizer.encode(
+                    '。')  # Handle both English and Chinese periods
+
+                # Then we iteration backwards until we find the first period
+                while tokens_new_context and tokens_new_context[-1] not in period_tokens:
+                    insertion_point -= 1
+                    tokens_new_context = tokens_context[:insertion_point]
+
+                # Insert the needle into the context at the found position
+                tokens_context = tokens_context[:insertion_point] + tokens_needle + tokens_context[insertion_point:]
+
+                # Log
+                insertion_percentage = (insertion_point / len(tokens_context)) * 100
+                self.insertion_percentages.append(insertion_percentage)
+                logger.debug(f"Inserted '{needle}' at {insertion_percentage:.2f}% of the context, "
+                             f'total length now: {len(tokens_context)} tokens')
+
+                # Adjust depth for next needle
+                depth_percent += depth_percent_interval
+
+        new_context = self.tokenizer.decode(tokens_context)
+        return new_context
+
+    def gen_prompt(self, input_d: dict, **kwargs) -> dict:
+        """
+        Generate the prompt for each sample in the dataset.
+        Args:
+            input_d: A dictionary containing the input data for the prompt.
+                It should contain 'context' and optionally 'question'.
+        Returns:
+            A dictionary containing the prompt data
+        """
+        context = input_d.get('context')
+        question = input_d.get('question')
+
+        prompt = self.prompt_template.format(context=context, question=question)
+
+        return self.gen_prompt_data(prompt, system_prompt=self.system_prompt)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        """
+        Parse the raw input labels (gold).
+        """
+        return input_d.get('answer', '').strip()
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = EvalType.CHECKPOINT) -> str:
+        """
+        Parse the predicted result and extract proper answer.
+        """
+        return result
+
+    def match(self, gold: str, pred: str) -> float:
+        """
+        Match the gold answer and the predicted answer.
+        """
+        from .utils import normalize_answer
+        norm_gold = normalize_answer(gold)
+        norm_pred = normalize_answer(pred)
+        # Use exact match for Needle in a Haystack
+        return exact_match(gold=norm_gold, pred=norm_pred)
+
+    def llm_match(self, gold: str, pred: str, judge: LLMJudge, **kwargs) -> dict:
+        """
+        Use LLM as a judge to evaluate the predicted answer against the gold answer.
+        """
+        from .utils import GENERAL_ORM_PROMPT, ORM_USER_TEMPLATE, parse_score
+
+        raw_input = kwargs.get('raw_input', None)
+        question = raw_input.get('question')
+        context_length = raw_input.get('context_length')
+        depth_percent = raw_input.get('depth_percent')
+
+        # get grading response
+        prompt = ORM_USER_TEMPLATE.format(question=question, gold=gold, pred=pred)
+        orm_response = judge(prompt=prompt, system_prompt=GENERAL_ORM_PROMPT)
+
+        # parse grading score with regex, [[score]]
+        score = parse_score(orm_response) if orm_response else 0.0
+        return {f'Context#{context_length} Depth#{depth_percent}': score}
+
+    def compute_metric(self, review_res_list: Union[List[dict], List[List[dict]]], **kwargs) -> List[dict]:
+        """
+        compute weighted mean of the bleu score of all samples
+
+        Args:
+            review_res_list: [score1, score2, ...]
+
+        Returns:
+            avg_res: List[dict]
+
+        """
+        items = super().compute_dict_metric(review_res_list, **kwargs)
+        return [{'metric_name': k, 'score': mean(v), 'num': len(v)} for k, v in items.items()]
+
+    def post_process_report(self, report: 'Report', **kwargs):
+        try:
+            import os
+
+            from .utils import draw_score_chat
+
+            report_path = kwargs.get('report_path')
+            data_frame = report.to_dataframe()
+            # split `Metric` to `Context` and `Depth`
+            data_frame[['Context', 'Depth']] = data_frame['Metric'].str.split(' ', n=1, expand=True)
+            data_frame['Depth'] = data_frame['Depth'].str.replace('Depth#', '').astype(float)
+            data_frame['Context'] = data_frame['Context'].str.replace('Context#', '').astype(int)
+            # split by `Subset` to multi sub data frame
+            for subset in data_frame['Subset'].unique():
+                sub_df = data_frame[data_frame['Subset'] == subset]
+                # draw charts for each subset
+                pivot_table = sub_df.pivot_table(
+                    values='Score', index=['Depth', 'Context'], aggfunc='mean').reset_index()
+                pivot_table = pivot_table.pivot(index='Depth', columns='Context', values='Score')
+                draw_score_chat(pivot_table, outpath=os.path.join(report_path, f'needle_haystack_heatmap_{subset}.png'))
+
+        except Exception as e:
+            logger.error(f'Error generating charts: {e}')

evalscope/benchmarks/needle_haystack/utils.py

@@ -0,0 +1,79 @@
+import matplotlib.pyplot as plt
+import os
+import re
+import seaborn as sns
+import string
+from matplotlib.colors import LinearSegmentedColormap
+
+
+def normalize_answer(s):
+
+    def remove_articles(text):
+        return re.sub(r'\b(a|an|the)\b', ' ', text)
+
+    def white_space_fix(text):
+        return ' '.join(text.split())
+
+    def remove_punc(text):
+        exclude = set(string.punctuation)
+        return ''.join(ch for ch in text if ch not in exclude)
+
+    def lower(text):
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(s))))
+
+
+def parse_score(score_str: str) -> int:
+    """
+    Parses a score string and returns an integer score.
+    The score should be in the format [[score]].
+    """
+    score_match = re.search(r'\[\[(\d+)\]\]', score_str)
+    if score_match:
+        score = int(score_match.group(1))
+        return score / 10.0
+    else:
+        return 0.0
+
+
+def draw_score_chat(pivot_table, outpath):
+    # Create a custom colormap. Go to https://coolors.co/ and pick cool colors
+    cmap = LinearSegmentedColormap.from_list('custom_cmap', ['#F0496E', '#EBB839', '#0CD79F'])
+
+    # Create the heatmap with better aesthetics
+    plt.figure(figsize=(17.5, 8))  # Can adjust these dimensions as needed
+    sns.heatmap(pivot_table, vmin=0.0, vmax=1.0, annot=True, fmt='.1f', cmap=cmap, cbar_kws={'label': 'Score'})
+
+    # More aesthetics
+    plt.title('Fact Retrieval Across Context Lengths ("Needle In A HayStack")')  # Adds a title
+    plt.xlabel('Token Limit')  # X-axis label
+    plt.ylabel('Depth Percent')  # Y-axis label
+    plt.xticks(rotation=45)  # Rotates the x-axis labels to prevent overlap
+    plt.yticks(rotation=0)  # Ensures the y-axis labels are horizontal
+    plt.tight_layout()  # Fits everything neatly into the figure area
+
+    # save the figure
+    plt.savefig(outpath, dpi=300, bbox_inches='tight')
+
+
+GENERAL_ORM_PROMPT = """You are an expert in verifying if the model answer is correct based on the reference answer.
+Your input is a question, a reference answer, and a model answer. You need to check if the model answer is correct based on the reference answer.
+You should focus on the correctness of the model answer compared to the reference answer, without attempting to solve the original question.
+You must provide your final score in the form of a number from 1 to 10, where:
+
+Score 1: The answer is completely unrelated to the reference.
+Score 3: The answer has minor relevance but does not align with the reference.
+Score 5: The answer has moderate relevance but contains inaccuracies.
+Score 7: The answer aligns with the reference but has minor omissions.
+Score 10: The answer is completely accurate and aligns perfectly with the reference.
+
+Only respond with a numberical score with formatted as [[score]]."""  # noqa: E501
+
+ORM_USER_TEMPLATE = """
+Question: {question}
+
+Reference Answer: {gold}
+
+Model Answer: {pred}
+"""

evalscope/benchmarks/simple_qa/simple_qa_adapter.py

@@ -148,6 +148,7 @@ class SimpleQAAdapter(DataAdapter):
             'is_correct': 1 if res == 'A' else 0,
             'is_incorrect': 1 if res == 'B' else 0,
             'is_not_attempted': 1 if res == 'C' else 0,
+            'judge_response': grading_response,
         }
 
     def compute_metric(self, review_res_list: List[dict], **kwargs) -> List[dict]: