evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/arena_hard/arena_hard_adapter.py

@@ -0,0 +1,149 @@
+# flake8: noqa: E501
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.metric import AggScore, SampleScore, Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+GRADER_SYSTEM_PROMPT = """Please act as an impartial judge and evaluate the quality of the responses provided by two AI assistants to the user prompt displayed below. You will be given assistant A's answer and assistant B's answer. Your job is to evaluate which assistant's answer is better.\n\nBegin your evaluation by generating your own answer to the prompt. You must provide your answers before judging any answers.\n\nWhen evaluating the assistants' answers, compare both assistants' answers with your answer. You must identify and correct any mistakes or inaccurate information.\n\nThen consider if the assistant's answers are helpful, relevant, and concise. Helpful means the answer correctly responds to the prompt or follows the instructions. Note when user prompt has any ambiguity or more than one interpretation, it is more helpful and appropriate to ask for clarifications or more information from the user than providing an answer based on assumptions. Relevant means all parts of the response closely connect or are appropriate to what is being asked. Concise means the response is clear and not verbose or excessive.\n\nThen consider the creativity and novelty of the assistant's answers when needed. Finally, identify any missing important information in the assistants' answers that would be beneficial to include when responding to the user prompt.\n\nAfter providing your explanation, you must output only one of the following choices as your final verdict with a label:\n\n1. Assistant A is significantly better: [[A>>B]]\n2. Assistant A is slightly better: [[A>B]]\n3. Tie, relatively the same: [[A=B]]\n4. Assistant B is slightly better: [[B>A]]\n5. Assistant B is significantly better: [[B>>A]]\n\nExample output: \"My final verdict is tie: [[A=B]]\"."""  # noqa: E501
+
+GRADER_TEMPLATE = """<|User Prompt|>\n{question}\n\n<|The Start of Assistant A's Answer|>\n{answer_1}\n<|The End of Assistant A's Answer|>\n\n<|The Start of Assistant B's Answer|>\n{answer_2}\n<|The End of Assistant B's Answer|>""".strip(
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='arena_hard',
+        pretty_name='ArenaHard',
+        tags=[Tags.INSTRUCTION_FOLLOWING, Tags.ARENA],
+        description=
+        'ArenaHard is a benchmark designed to evaluate the performance of large language models in a competitive setting, '
+        'where models are pitted against each other in a series of tasks to determine their relative strengths and weaknesses. '
+        'It includes a set of challenging tasks that require reasoning, understanding, and generation capabilities. '
+        'Currently not support `style-controlled winrate`; the official Judge model is `gpt-4-1106-preview`, while the baseline model is `gpt-4-0314`.',
+        dataset_id='AI-ModelScope/arena-hard-auto-v0.1',
+        metric_list=['winrate'],
+        aggregation='elo',
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template='{question}'
+    )
+)
+class ArenaHardAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True  # Use LLM as a judge by default
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a data record to a Sample object.
+
+        Args:
+            record (Dict[str, Any]): Input data record.
+
+        Returns:
+            Sample: Sample object with input, target, and metadata.
+        """
+        question = record['question']
+        baseline_prediction = record['prediction']  # baseline model prediction
+
+        return Sample(
+            input=question, target=baseline_prediction, metadata={'capability': record.get('capability', 'unknown')}
+        )
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        from .utils import get_judge_score, post_process_arenahard
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        question = task_state.input_text
+
+        # reference is baseline answer 'A', filtered_prediction is model answer 'B'
+        prompt1 = GRADER_TEMPLATE.format(question=question, answer_1=reference, answer_2=filtered_prediction)
+        # reverse the order
+        prompt2 = GRADER_TEMPLATE.format(question=question, answer_1=filtered_prediction, answer_2=reference)
+
+        # get grading response
+        game1_response = self.llm_judge.judge(prompt1, system_prompt=GRADER_SYSTEM_PROMPT)
+        game2_response = self.llm_judge.judge(prompt2, system_prompt=GRADER_SYSTEM_PROMPT)
+
+        # parse grading response
+        res1 = post_process_arenahard(game1_response)
+        res2 = post_process_arenahard(game2_response)
+
+        score1 = get_judge_score(res1, reverse=True)
+        score2 = get_judge_score(res2, reverse=False)
+
+        battle_result = {
+            'model_a':
+            'gpt4-0314',
+            'model_b':
+            'test_model',
+            'games': [
+                {
+                    'user_prompt': prompt1,
+                    'judgment': game1_response,
+                    'score': res1
+                },
+                {
+                    'user_prompt': prompt2,
+                    'judgment': game2_response,
+                    'score': res2
+                },
+            ]
+        }
+
+        # Set score based on the battle result
+        score.value = {'score': (score1 + score2) / 2}
+        score.explanation = f'LLM judge battles: Game1: {game1_response[:100]}... Game2: {game2_response[:100]}...'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id,
+            'battle_result': battle_result
+        }
+        return score
+
+    def aggregate_scores(self, sample_scores: List[SampleScore]) -> List[AggScore]:
+        import pandas as pd
+
+        from .utils import compute_mle_elo, get_battles_from_row, get_bootstrap_result, get_win_rate_column
+
+        battles = pd.concat([get_battles_from_row(res.score.metadata['battle_result']) for res in sample_scores])
+
+        bootstrap_online_elo = compute_mle_elo(battles)
+
+        stats = pd.DataFrame()
+        stats['results'] = None
+        stats['results'] = stats['results'].astype('object')
+
+        for i, model in enumerate(bootstrap_online_elo.index):
+            # assert model in bootstrap_elo_lu.columns
+            stats.at[i, 'model'] = model
+            stats.at[i, 'score'] = bootstrap_online_elo[model]
+
+        score = get_win_rate_column(stats, 'score', 'gpt4-0314').at['test_model']
+
+        return [AggScore(
+            score=score,
+            metric_name='winrate',
+            num=len(sample_scores),
+        )]

evalscope/benchmarks/arena_hard/utils.py

@@ -0,0 +1,186 @@
+import math
+import numpy as np
+import pandas as pd
+import re
+from collections import defaultdict
+from sklearn.linear_model import LogisticRegression
+from tqdm import tqdm
+
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def post_process_arenahard(completion):
+    result = re.findall(r'\[\[([AB<>=]+)\]\]', completion)
+    if result:
+        return result[0]
+    else:
+        return None
+
+
+def get_judge_score(result, reverse=False):
+    """
+    Calculate the judge score, considering confidence weight.
+
+    Args:
+        result: Judgment result ('A=B', 'A>B', 'A>>B', 'B>A', 'B>>A')
+        reverse: Whether to reverse the score
+
+    Returns:
+        float: Weighted score
+    """
+
+    # Base score mapping - using finer-grained scores
+    if not reverse:
+        score_mapping = {
+            'A=B': 0.5,  # Tie
+            'A>B': 0.75,  # A slightly wins
+            'A>>B': 1.0,  # A significantly wins
+            'B>A': 0.25,  # B slightly wins
+            'B>>A': 0.0,  # B significantly wins
+        }
+    else:
+        score_mapping = {
+            'A=B': 0.5,  # Tie
+            'A>B': 0.25,  # A slightly wins
+            'A>>B': 0.0,  # A significantly wins
+            'B>A': 0.75,  # B slightly wins
+            'B>>A': 1.0,  # B significantly wins
+        }
+
+    base_score = score_mapping.get(result, 0.5)
+
+    return base_score
+
+
+def get_battles_from_row(row, first_game_only=False, multiplier=3):
+    results = []
+    output = {'model_a': row['model_a'], 'model_b': row['model_b']}
+
+    game = row['games'][0]
+    weight = 1
+    if game['score'] == 'A=B':
+        output['winner'] = 'tie'
+    elif game['score'] == 'A>B':
+        output['winner'] = 'model_a'
+    elif game['score'] == 'A>>B':
+        output['winner'] = 'model_a'
+        weight = multiplier
+    elif game['score'] == 'B>A':
+        output['winner'] = 'model_b'
+    elif game['score'] == 'B>>A':
+        output['winner'] = 'model_b'
+        weight = multiplier
+    else:
+        weight = 0
+
+    if weight:
+        results += [output] * weight
+
+    if first_game_only:
+        return pd.DataFrame(results)
+
+    # game 2
+    output = {'model_a': row['model_a'], 'model_b': row['model_b']}
+
+    game = row['games'][1]
+
+    weight = 1
+    if game['score'] == 'A=B':
+        output['winner'] = 'tie'
+    elif game['score'] == 'A>B':
+        output['winner'] = 'model_b'
+    elif game['score'] == 'A>>B':
+        output['winner'] = 'model_b'
+        weight = multiplier
+    elif game['score'] == 'B>A':
+        output['winner'] = 'model_a'
+    elif game['score'] == 'B>>A':
+        output['winner'] = 'model_a'
+        weight = multiplier
+    else:
+        weight = 0
+
+    if weight:
+        results += [output] * weight
+
+    return pd.DataFrame(results)
+
+
+def compute_mle_elo(df, SCALE=400, BASE=10, INIT_RATING=1000):
+    models = pd.concat([df['model_a'], df['model_b']]).unique()
+    models = pd.Series(np.arange(len(models)), index=models)
+
+    # duplicate battles
+    df = pd.concat([df, df], ignore_index=True)
+    p = len(models.index)
+    n = df.shape[0]
+
+    X = np.zeros([n, p])
+    X[np.arange(n), models[df['model_a']]] = +math.log(BASE)
+    X[np.arange(n), models[df['model_b']]] = -math.log(BASE)
+
+    # one A win => two A win
+    Y = np.zeros(n)
+    Y[df['winner'] == 'model_a'] = 1.0
+
+    # one tie => one A win + one B win
+    # find tie + tie (both bad) index
+    tie_idx = (df['winner'] == 'tie') | (df['winner'] == 'tie (bothbad)')
+    tie_idx[len(tie_idx) // 2:] = False
+    Y[tie_idx] = 1.0
+
+    if len(np.unique(Y)) < 2:
+        logger.info('Warning: Only one class in the data')
+        elo_scores = pd.Series(INIT_RATING, index=models.index)
+        if np.all(Y == 1.0):
+            elo_scores[df['model_a'].iloc[0]] += SCALE  # Boost the winning model
+        elif np.all(Y == 0.0):
+            elo_scores[df['model_b'].iloc[0]] += SCALE  # Boost the winning model
+        return elo_scores.sort_values(ascending=False)
+
+    lr = LogisticRegression(
+        fit_intercept=False, penalty=None, tol=1e-8
+    )  # May need to set a small value when not use GPT4 as judge model
+    lr.fit(X, Y)
+
+    elo_scores = SCALE * lr.coef_[0] + INIT_RATING
+
+    # set anchor as gpt4-0314 = 1000
+    if 'gpt4-0314' in models.index:
+        elo_scores += 1000 - elo_scores[models['gpt4-0314']]
+    return pd.Series(elo_scores, index=models.index).sort_values(ascending=False)
+
+
+def get_bootstrap_result(battles, func_compute_elo, num_round):
+    rows = []
+    for _ in tqdm(range(num_round), desc='bootstrap'):
+        res = func_compute_elo(battles.sample(frac=1.0, replace=True))
+        if res is not None:
+            rows.append(res)
+    df = pd.DataFrame(rows)
+    return df[df.median().sort_values(ascending=False).index]
+
+
+def predict_win_rate(elo_ratings, SCALE=400, BASE=10, INIT_RATING=1000):
+    names = sorted(list(elo_ratings.keys()))
+    wins = defaultdict(lambda: defaultdict(lambda: 0))
+    for a in names:
+        for b in names:
+            ea = 1 / (1 + BASE**((elo_ratings[b] - elo_ratings[a]) / SCALE))
+            wins[a][b] = ea
+            wins[b][a] = 1 - ea
+
+    data = {a: [wins[a][b] if a != b else np.NAN for b in names] for a in names}
+
+    df = pd.DataFrame(data, index=names)
+    df.index.name = 'model_a'
+    df.columns.name = 'model_b'
+    return df.T
+
+
+def get_win_rate_column(df, column, baseline='gpt4-0314'):
+    to_dict = df[['model', column]].set_index('model').to_dict()[column]
+    win_rate_table = predict_win_rate(to_dict)
+    return win_rate_table[baseline].fillna(0.5).apply(lambda x: round(x, 4))

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -1,19 +1,16 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
 
-import json
 import os
-import random
 import re
+from typing import Any, Dict
 
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.constants import AnswerKeys
-from evalscope.metrics import AverageAccuracy, exact_match
-from evalscope.models.chat_adapter import ChatGenerationModelAdapter
-from evalscope.utils import ResponseParser
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
-# flake8: noqa
-
 logger = get_logger()
 
 # BBH multiple choice subset list
@@ -57,185 +54,148 @@ FREE_FORM_LIST = [
 TASK_TYPE = 'task_type'
 SUBSET_LIST = MULTIPLE_CHOICE_LIST + FREE_FORM_LIST
 
-
-@Benchmark.register(
-    name='bbh',
-    dataset_id='modelscope/bbh',
-    model_adapter=ChatGenerationModelAdapter,
-    subset_list=SUBSET_LIST,
-    metric_list=[AverageAccuracy],
-    few_shot_num=3,
-    train_split=None,
-    eval_split='test',
-    prompt_template='',
+PROMPT_TEMPLATE = """
+Q: {question}
+A: Let's think step by step. Put your final answer in the format of "So the answer is $ANSWER" (without quotes and markdown) where $ANSWER is the answer to the problem.
+""".lstrip()  # noqa: E501
+
+FEWSHOT_TEMPLATE = """
+{fewshot}
+
+""".lstrip() + PROMPT_TEMPLATE
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='bbh',
+        pretty_name='BBH',
+        dataset_id='evalscope/bbh',
+        tags=[Tags.REASONING],
+        description=
+        'The BBH (Big Bench Hard) benchmark is a collection of challenging tasks designed to evaluate the reasoning capabilities of AI models. It includes both free-form and multiple-choice tasks, covering a wide range of reasoning skills.',  # noqa: E501
+        subset_list=SUBSET_LIST,
+        few_shot_num=3,
+        train_split=None,
+        eval_split='test',
+        metric_list=['acc'],
+        prompt_template=PROMPT_TEMPLATE,
+        few_shot_prompt_template=FEWSHOT_TEMPLATE,
+    )
 )
-class BBHAdapter(DataAdapter):
+class BBHAdapter(DefaultDataAdapter):
     """
     Adapter for BBH free-form and multiple-choices sub-tasks.
     """
 
     def __init__(self, **kwargs):
-
         few_shot_num = kwargs.get('few_shot_num', 3)
 
         if few_shot_num != 3 and few_shot_num != 0:
-            logger.error(f'BBH uses 3-shot examples with CoT or 0-shot by system, but got {few_shot_num}. '
-                         f'Use 3-shot by default.')
+            logger.error(
+                f'BBH uses 3-shot examples with CoT or 0-shot by system, but got {few_shot_num}. '
+                f'Use 3-shot by default.'
+            )
             kwargs['few_shot_num'] = 3
 
         super().__init__(**kwargs)
 
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            for split_name in [self.eval_split]:
-                if os.path.exists(dataset_name_or_path):
-                    file_path = os.path.join(dataset_name_or_path, f'{subset_name}.json')
-                else:
-                    file_path: str = os.path.join(work_dir, dataset_name_or_path, f'{subset_name}.json')
-                if os.path.exists(file_path):
-                    with open(file_path, 'r') as f:
-                        examples = json.load(f)['examples']
-                        if subset_name in data_dict:
-                            data_dict[subset_name].update({split_name: examples})
-                        else:
-                            data_dict[subset_name] = {split_name: examples}
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate model prompt from raw data, unify the prompt format for bbh(multiple choice) benchmark.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the BBH:
-
-            {
-                'input': '((-1 + 2 + 9 * 5) - (-2 + -4 + -4 * -7)) =',
-                'target': '24',
-            }
-
-        Returns:
-            {'data': ['xxx']}
-        """
-        # few_shot_list: should be ['xxxx']
-        cot_prompts: str = few_shot_list[0] if len(few_shot_list) > 0 else ''
-        full_prompt: str = f"Follow the given examples and answer the question.\n{cot_prompts}\n\nQ: {input_d['input']}\nA: Let's think step by step."
-
-        return {'data': [full_prompt], 'system_prompt': self.prompt_template}
-
-    def gen_prompts(self, data_dict: dict) -> dict:
-        """
-        Generate dataset prompts from raw input, unify the prompt format for different datasets.
-
-        Args:
-            data_dict:  Refer to the output of load method: evalscope.benchmarks.benchmark.Benchmark.load
-
-        Returns:
-            {'subset_name': [prompt_d_1, prompt_d_2, ...]}
-            prompt_d_i (dict): refer to the output of gen_prompt method.
-
-        e.g. train -- few-shot data, test -- target dataset to evaluate.
-        """
-        res_dict: dict = {}
-
-        if self.few_shot_num < 0:
-            raise ValueError(f'Invalid shot_num: {self.few_shot_num} for few-shot evaluation.')
-
-        logger.info(f'Use default settings: '
-                    f'> few_shot_num: {self.few_shot_num}, '
-                    f'> few_shot_split: {self.train_split}, '
-                    f'> target_eval_split: {self.eval_split}')
-
-        for sub_name, sub_data_dict in data_dict.items():
-            few_shot_data = []
-            if self.few_shot_num > 0:
-                with open(os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{sub_name}.txt'), 'r') as f:
-                    cot_prompt_str = f.read()
-                few_shot_data = [cot_prompt_str]
-
-            res_dict[sub_name] = []
-            for sample_d in sub_data_dict[self.eval_split]:
-                prompt_d = self.gen_prompt(input_d=sample_d, few_shot_list=few_shot_data)
-                sample_d_new = sample_d.copy()
-                if sub_name in MULTIPLE_CHOICE_LIST:
-                    sample_d_new[TASK_TYPE] = MULTIPLE_CHOICE
-                elif sub_name in FREE_FORM_LIST:
-                    sample_d_new[TASK_TYPE] = FREE_FORM
-                else:
-                    raise ValueError(f'Invalid subset name: {sub_name}')
-
-                prompt_d[AnswerKeys.RAW_INPUT] = sample_d_new
-                res_dict[sub_name].append(prompt_d)
-
-        rnd = random.Random()
-        rnd.seed(42)
-        for k, v in res_dict.items():
-            rnd.shuffle(v)
-
-        return res_dict
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        # Get the gold choice
-        gold = input_d.get('target')
-        if gold is None:
-            logger.error(f'BBHAdapter: gold is None.')
-        return gold
-
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        """
-        Parse the model output to get the answer. Could be the best choice index.
-
-        Args:
-            result: Predicted answer from the model. Usually a string for chat.
-            raw_input_d (dict): The raw input. Depending on the dataset.
-            eval_type: 'checkpoint' or 'service' or `custom`, default: 'checkpoint'
-
-        Returns:
-            The parsed answer. Depending on the dataset. Usually a string for chat.
-        """
-        # Note: to use same extraction method for both of checkpoint/service/custom.
-        task_type: str = raw_input_d.get(TASK_TYPE)
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        input = record['input']
+        target = record['target'].replace('(', '').replace(')', '').strip()  # Clean up the target answer
+
+        # Determine task type based on subset name
+        task_type = None
+        subset_name = self.current_subset_name
+        if subset_name in MULTIPLE_CHOICE_LIST:
+            task_type = MULTIPLE_CHOICE
+        elif subset_name in FREE_FORM_LIST:
+            task_type = FREE_FORM
+
+        metadata = {TASK_TYPE: task_type}
+
+        return Sample(input=input, target=target, metadata=metadata, subset_key=subset_name)
+
+    def format_fewshot_template(self, fewshot: str, sample: Sample) -> str:
+        # Load CoT prompts from file for BBH
+        subset_name = sample.subset_key
+        if subset_name:
+            cot_file_path = os.path.join(os.path.dirname(__file__), 'cot_prompts', f'{subset_name}.txt')
+            if os.path.exists(cot_file_path):
+                with open(cot_file_path, 'r', encoding='utf-8') as f:
+                    fewshot = f.read().strip()
+        return self.few_shot_prompt_template.format(
+            fewshot=fewshot,
+            question=sample.input,
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState):
+        task_type = task_state.metadata.get(TASK_TYPE)
 
         if task_type == MULTIPLE_CHOICE:
-            return self._extract_mc_answer(result)
+            return self._extract_mc_answer(prediction)
         elif task_type == FREE_FORM:
-            return self._extract_ff_answer(result)
+            return self._extract_ff_answer(prediction)
         else:
-            raise ValueError(f'Invalid task type: {task_type}')
-
-    def match(self, gold: str, pred: str) -> float:
-        return exact_match(gold=gold, pred=pred)
+            return prediction.strip()
 
     @classmethod
     def _extract_mc_answer(cls, ans: str) -> str:
         """
-        Extract the answer from the model output for Multiple choice task.
+        Extract normalized answer for BBH multiple-choice tasks.
+        Handles formats like:
+        - "answer is (A)"
+        - "The answer is A."
+        - Extra text after answer.
+        Always uses the *last* occurrence of "answer is".
         """
-        ans_line = ans.split('answer is ')
-        if len(ans_line) != 1:
-            ans = ans_line[1].strip()
-        match = re.search(r'\(([A-Z])\)*', ans)
+        ans = ans.strip()
+
+        parts = ans.split('So the answer is ')
+        if len(parts) > 1:
+            ans = parts[-1].strip()
+        ans = ans.split('\n')[0].strip()
+
+        # Remove trailing period
+        if ans.endswith('.'):
+            ans = ans[:-1].strip()
+
+        # Capture uppercase letter inside parentheses (A) (B) ...
+        match = re.search(r'\(([A-Z])\)', ans)
         if match:
             return match.group(1)
-        match = re.search(r'([A-Z])', ans)
+
+        # Capture single uppercase letter
+        match = re.search(r'\b([A-Z])\b', ans)
         if match:
             return match.group(1)
+
         return ans
 
     @classmethod
     def _extract_ff_answer(cls, ans: str):
         """
-        Extract the answer from the model output for Free-form task.
+        Extract the normalized answer for BBH free-form tasks.
+        Handles patterns like:
+        - "answer is XXX."
+        - "The answer is **valid**."
+        - Extra trailing dots / line breaks.
+        - Bold-marked answers (**xxx**).
+        Always uses the *last* occurrence of "answer is".
         """
-        res = ResponseParser.parse_first_option(ans)
-        if res:
-            return res
-
-        ans_line = ans.split('answer is ')
-        if len(ans_line) != 1:
-            ans = ans_line[1].strip()
-        ans = ans.split('\n')[0]
+        ans = ans.strip()
+
+        parts = ans.split('So the answer is ')
+        if len(parts) > 1:
+            ans = parts[-1].strip()
+        ans = ans.split('\n')[0].strip()
+
+        # Remove trailing period
         if ans.endswith('.'):
-            ans = ans[:-1]
+            ans = ans[:-1].strip()
+
+        # If answer is in bold (**xxx**), prefer the content inside
+        match = re.search(r'\*\*(.*?)\*\*', ans)
+        if match:
+            ans = match.group(1).strip()
+
         return ans