evalscope 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

evalscope/api/mixin/sandbox_mixin.py

@@ -0,0 +1,204 @@
+import asyncio
+import threading
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from ms_enclave.sandbox.manager import SandboxManager
+
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class SandboxMixin:
+    """Sandbox mixin for sandboxed code execution."""
+
+    def __init__(self, task_config: 'TaskConfig'):
+        self._task_config = task_config
+
+        self._manager: Optional['SandboxManager'] = None
+        """Sandbox manager instance."""
+
+        self._sandbox_id: Optional[str] = None
+        """Sandbox ID."""
+
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        """Event loop for async operations."""
+
+        # Initialize sandbox synchronously by running async methods
+        if self.use_sandbox:
+            self._loop = asyncio.new_event_loop()
+
+            # Start the loop in a separate thread
+            def run_loop():
+                asyncio.set_event_loop(self._loop)
+                self._loop.run_forever()
+
+            self._loop_thread = threading.Thread(target=run_loop, daemon=True)
+            self._loop_thread.start()
+
+            # Wait for initialization
+            future = asyncio.run_coroutine_threadsafe(self._async_init(), self._loop)
+            future.result()
+
+        super().__init__()
+
+    async def _async_init(self):
+        """Async initialization helper."""
+        await self.init_sandbox_manager_async()
+        await self.init_sandbox_async()
+
+    @property
+    def use_sandbox(self) -> bool:
+        """
+        Return whether to use sandbox for the benchmark.
+        """
+        if not self._task_config:
+            return False
+        else:
+            return self._task_config.use_sandbox
+
+    @property
+    def sandbox_manager(self) -> Optional['SandboxManager']:
+        """Get the sandbox manager instance."""
+        return self._manager
+
+    @property
+    def sandbox_id(self) -> Optional[str]:
+        """Get the sandbox ID."""
+        return self._sandbox_id
+
+    async def init_sandbox_manager_async(self) -> Optional['SandboxManager']:
+        """Initialize the sandbox manager asynchronously."""
+        if self._manager is not None:
+            return self._manager
+
+        if not self.use_sandbox:
+            return None
+
+        from ms_enclave.sandbox.manager import HttpSandboxManager, LocalSandboxManager
+
+        manager_config = self._task_config.sandbox_manager_config or {}
+        if manager_config.get('base_url'):
+            # Remote manager
+            self._manager = HttpSandboxManager(**manager_config)
+        else:
+            # Local manager
+            self._manager = LocalSandboxManager(**manager_config)
+
+        await self._manager.start()
+        logger.info('Sandbox manager initialized.')
+        return self._manager
+
+    def init_sandbox_manager(self) -> Optional['SandboxManager']:
+        """Initialize the sandbox manager."""
+        if self._manager is not None:
+            return self._manager
+
+        if not self.use_sandbox:
+            return None
+
+        # Use the dedicated loop if available
+        if self._loop and not self._loop.is_closed():
+            future = asyncio.run_coroutine_threadsafe(self.init_sandbox_manager_async(), self._loop)
+            return future.result()
+        else:
+            # Fallback for cases where no loop is available
+            return asyncio.run(self.init_sandbox_manager_async())
+
+    async def init_sandbox_async(self) -> Optional[str]:
+        """Initialize the sandbox instance asynchronously."""
+        if self._sandbox_id is not None:
+            return self._sandbox_id
+
+        if not self.use_sandbox:
+            return None
+
+        from ms_enclave.sandbox.model import DockerSandboxConfig, SandboxType
+
+        sandbox_config = self._task_config.sandbox_config or DockerSandboxConfig(
+            image='python:3.11-slim', tools_config={
+                'shell_executor': {},
+                'python_executor': {}
+            }
+        )
+        sandbox_type = self._task_config.sandbox_type or SandboxType.DOCKER
+
+        self._sandbox_id = await self._manager.create_sandbox(sandbox_type=sandbox_type, config=sandbox_config)
+
+        sandbox_info = await self._manager.get_sandbox_info(self._sandbox_id)
+
+        logger.info(f'Sandbox of type {sandbox_type} initialized. Info: {sandbox_info.model_dump(exclude_none=True)}')
+        return self._sandbox_id
+
+    def init_sandbox(self) -> Optional[str]:
+        """Initialize the sandbox instance."""
+        if self._sandbox_id is not None:
+            return self._sandbox_id
+
+        if not self.use_sandbox:
+            return None
+
+        # Use the dedicated loop if available
+        if self._loop and not self._loop.is_closed():
+            future = asyncio.run_coroutine_threadsafe(self.init_sandbox_async(), self._loop)
+            return future.result()
+        else:
+            # Fallback for cases where no loop is available
+            return asyncio.run(self.init_sandbox_async())
+
+    def execute_code_in_sandbox(self, code: str, timeout: int = 60, language: str = 'python') -> Dict[str, Any]:
+        """Execute code in the sandbox."""
+        if not self._sandbox_id or not self._manager:
+            logger.warning('Sandbox is not initialized.')
+            return {'error': 'Sandbox is not initialized.'}
+
+        from ms_enclave.sandbox.model import ExecutionStatus, ToolResult
+
+        async def _execute_async():
+            if language.lower() == 'python':
+                tool_name = 'python_executor'
+                parameters = {'code': code, 'timeout': timeout}
+                result = await self._manager.execute_tool(self._sandbox_id, tool_name, parameters)
+            elif language.lower() == 'shell':
+                tool_name = 'shell_executor'
+                parameters = {'command': code, 'timeout': timeout}
+                result = await self._manager.execute_tool(self._sandbox_id, tool_name, parameters)
+            else:
+                logger.warning(f"Unsupported language: {language}. Supported languages are 'python' and 'shell'.")
+                result = ToolResult(
+                    status=ExecutionStatus.ERROR,
+                    tool_name='code_executor',
+                    output=f"Unsupported language: {language}. Supported languages are 'python' and 'shell'."
+                )
+            return result
+
+        # Use the dedicated loop if available
+        if self._loop and not self._loop.is_closed():
+            future = asyncio.run_coroutine_threadsafe(_execute_async(), self._loop)
+            result = future.result(timeout + 10)  # Add some buffer to the timeout
+        else:
+            # Fallback for cases where no loop is available
+            result = asyncio.run(_execute_async())
+
+        return result.model_dump(exclude_none=True)
+
+    def sandbox_finalize(self, *args, **kwargs):
+        """Finalize the sandbox manager."""
+        if self._manager:
+            try:
+                if self._loop and not self._loop.is_closed():
+                    # Stop the manager using the dedicated loop
+                    future = asyncio.run_coroutine_threadsafe(self._manager.stop(), self._loop)
+                    future.result(timeout=30)
+
+                    # Stop the event loop
+                    self._loop.call_soon_threadsafe(self._loop.stop)
+                    if hasattr(self, '_loop_thread'):
+                        self._loop_thread.join(timeout=5)
+
+                logger.info('Sandbox manager finalized.')
+            except Exception as e:
+                logger.warning(f'Error finalizing sandbox manager: {e}')

evalscope/api/model/generate_config.py

@@ -25,9 +25,7 @@ class ResponseSchema(BaseModel):
 
 class GenerateConfig(BaseModel):
     """Model generation options."""
-
-    max_retries: Optional[int] = Field(default=None)
-    """Maximum number of times to retry request (defaults to unlimited)."""
+    model_config = {'extra': 'allow'}
 
     timeout: Optional[int] = Field(default=None)
     """Request timeout (in seconds)."""
@@ -38,9 +36,6 @@ class GenerateConfig(BaseModel):
     stream: Optional[bool] = Field(default=None)
     """Whether to stream the response (default is model specific)."""
 
-    system_message: Optional[str] = Field(default=None)
-    """Override the default system message."""
-
     max_tokens: Optional[int] = Field(default=None)
     """The maximum number of tokens that can be generated in the completion (default is model specific)."""
 

evalscope/api/model/model.py

@@ -318,7 +318,7 @@ def get_model_with_task_config(task_config: 'TaskConfig') -> Model:
 
 @thread_safe
 def get_model(
-    model: str,
+    model: Union[str, Model, ModelAPI],
     eval_type: str,
     base_url: Optional[str] = None,
     api_key: Optional[str] = None,
@@ -346,6 +346,9 @@ def get_model(
     if isinstance(model, Model):
         return model
 
+    if isinstance(model, ModelAPI):
+        return Model(model, config, model_args)
+
     # see if we can return a memoized model instance
     # (exclude mockllm since custom_outputs is an infinite generator)
     model_cache_key: str = ''
@@ -362,7 +365,7 @@ def get_model(
 
     logger.info(
         f'Creating model {model} with eval_type={eval_type} '
-        f'base_url={base_url}, api_key={api_key}, config={config}, model_args={model_args}'
+        f'base_url={base_url}, config={config.model_dump(exclude_none=True)}, model_args={model_args}'
     )
 
     # find a matching model type

evalscope/api/tool/tool_info.py

@@ -1,7 +1,7 @@
 import inspect
 from dataclasses import dataclass
 from docstring_parser import Docstring, parse
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from typing import Any, Callable, Dict, List, Literal, Optional, TypeAlias, Union, get_args, get_type_hints
 
 from evalscope.utils.json_schema import JSONSchema, JSONType, json_schema, python_type_to_json_type

evalscope/app/app.py

@@ -6,6 +6,7 @@ import argparse
 from evalscope.utils.logger import configure_logging
 from .arguments import add_argument
 from .ui import create_app_ui
+from .utils.env_utils import setup_env
 
 
 def create_app(args: argparse.Namespace):
@@ -17,6 +18,8 @@ def create_app(args: argparse.Namespace):
     """
     configure_logging(debug=args.debug)
 
+    setup_env(args)
+
     demo = create_app_ui(args)
 
     demo.launch(

evalscope/app/ui/single_model.py

@@ -198,9 +198,9 @@ def create_single_model_tab(sidebar: 'SidebarComponents', lang: str):
 
         # Process the data for display
         input_md = row['Input'] + '\n\n' + process_model_prediction(row['Metadata'])
-        generated_md = process_model_prediction(row['Generated'])
-        gold_md = process_model_prediction(row['Gold'])
-        pred_md = convert_markdown_image(process_model_prediction(row['Pred']))
+        generated_md = convert_markdown_image(row['Generated'])
+        gold_md = convert_markdown_image(row['Gold'])
+        pred_md = process_model_prediction(row['Pred'])
         score_md = process_json_content(row['Score'])
         nscore_val = float(row['NScore']) if not pd.isna(row['NScore']) else 0.0
 

evalscope/app/utils/data_utils.py

@@ -2,7 +2,6 @@
 Data loading and processing utilities for the Evalscope dashboard.
 """
 import glob
-import numpy as np
 import os
 import pandas as pd
 from typing import Any, Dict, List, Union
@@ -160,17 +159,18 @@ def get_model_prediction(work_dir: str, model_name: str, dataset_name: str, subs
             if f'{sample_dataset_name}/{sample_subset_name}' != subset_name:
                 continue
 
-        prediction = sample_score.score.prediction
-        target = review_result.target
-        extracted_prediction = sample_score.score.extracted_prediction
         score = sample_score.score
+        metadata = sample_score.sample_metadata
+        prediction = score.prediction
+        target = review_result.target
+        extracted_prediction = score.extracted_prediction
         raw_d = {
             'Index': str(review_result.index),
             'Input': review_result.input.replace('\n', '\n\n'),  # for markdown
-            'Metadata': sample_score.sample_metadata,
-            'Generated': prediction if prediction != extracted_prediction else '*Same as Pred*',
+            'Metadata': metadata,
+            'Generated': prediction,
             'Gold': target,
-            'Pred': extracted_prediction,
+            'Pred': extracted_prediction if extracted_prediction != prediction else '*Same as Generated*',
             'Score': score.model_dump(exclude_none=True),
             'NScore': normalize_score(score.main_value)
         }

evalscope/app/utils/env_utils.py

@@ -0,0 +1,12 @@
+# flake8: noqa
+import os
+
+
+def setup_env(args):
+    compat_dsw_gradio(args)
+
+
+def compat_dsw_gradio(args) -> None:
+    if ('JUPYTER_NAME' in os.environ) and ('dsw-'
+                                           in os.environ['JUPYTER_NAME']) and ('GRADIO_ROOT_PATH' not in os.environ):
+        os.environ['GRADIO_ROOT_PATH'] = f"/{os.environ['JUPYTER_NAME']}/proxy/{args.server_port}"

evalscope/app/utils/text_utils.py

@@ -2,11 +2,9 @@
 Text processing utilities for the Evalscope dashboard.
 """
 import json
-import numpy as np
 import os
-import pandas as pd
 import re
-from typing import Any, Dict, List
+from typing import Any, Dict, List, Optional
 
 from evalscope.utils.logger import get_logger
 from ..constants import LATEX_DELIMITERS
@@ -14,15 +12,19 @@ from ..constants import LATEX_DELIMITERS
 logger = get_logger()
 
 
-def convert_markdown_image(text):
-    if not os.path.isfile(text):
-        return text
-    # Convert the image path to a markdown image tag
-    if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
-        text = os.path.abspath(text)
-        image_tag = f'![image](gradio_api/file={text})'
-        logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+def convert_markdown_image(text: str):
+    if text.startswith('data:image'):
+        # Convert base64 image data to a markdown image tag
+        image_tag = f'![image]({text})'
+        logger.debug(f'Converting base64 image data to markdown: {text[:30]}... -> {image_tag[:40]}...')
         return image_tag
+    elif os.path.isfile(text):
+        # Convert the image path to a markdown image tag
+        if text.endswith('.png') or text.endswith('.jpg') or text.endswith('.jpeg'):
+            text = os.path.abspath(text)
+            image_tag = f'![image](gradio_api/file={text})'
+            logger.debug(f'Converting image path to markdown: {text} -> {image_tag}')
+            return image_tag
     return text
 
 
@@ -85,7 +87,7 @@ def process_model_prediction_old(item: Any, max_length: int = 2048) -> str:
     return result
 
 
-def process_model_prediction(item: Any, max_length: int = 32000) -> str:
+def process_model_prediction(item: Any, max_length: Optional[int] = None) -> str:
     if isinstance(item, (dict, list)):
         result = json.dumps(item, ensure_ascii=False, indent=2)
         result = f'```json\n{result}\n```'

evalscope/arguments.py

@@ -2,7 +2,7 @@
 import argparse
 import json
 
-from evalscope.constants import EvalBackend, EvalType, JudgeStrategy, ModelTask, OutputType
+from evalscope.constants import EvalBackend, EvalType, JudgeStrategy, ModelTask
 
 
 class ParseStrArgsAction(argparse.Action):
@@ -60,8 +60,7 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--generation-config', type=str, action=ParseStrArgsAction, help='The generation config, should be a string.')  # noqa: E501
 
     # Evaluation-related arguments
-    parser.add_argument('--eval-type', type=str, help='The type for evaluating.',
-                        choices=[EvalType.CHECKPOINT, EvalType.CUSTOM, EvalType.SERVICE])
+    parser.add_argument('--eval-type', type=str, help='The type for evaluating.')
     parser.add_argument('--eval-backend', type=str, help='The evaluation backend to use.',
                         choices=[EvalBackend.NATIVE, EvalBackend.OPEN_COMPASS, EvalBackend.VLM_EVAL_KIT, EvalBackend.RAG_EVAL])  # noqa: E501
     parser.add_argument('--eval-config', type=str, required=False, help='The eval task config file path for evaluation backend.')  # noqa: E501
@@ -77,7 +76,6 @@ def add_argument(parser: argparse.ArgumentParser):
     # Debug and runtime mode arguments
     parser.add_argument('--ignore-errors', action='store_true', default=False, help='Ignore errors during evaluation.')
     parser.add_argument('--debug', action='store_true', default=False, help='Debug mode, will print information for debugging.')  # noqa: E501
-    parser.add_argument('--dry-run', action='store_true', default=False, help='Dry run in single processing mode.')
     parser.add_argument('--seed', type=int, default=42, help='Random seed for reproducibility.')
     parser.add_argument('--api-key', type=str, default='EMPTY', help='The API key for the remote API model.')
     parser.add_argument('--api-url', type=str, default=None, help='The API url for the remote API model.')
@@ -89,6 +87,12 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--judge-model-args', type=json.loads, default='{}', help='The judge model args, should be a json string.')  # noqa: E501
     parser.add_argument('--judge-worker-num', type=int, default=1, help='The number of workers for the judge model.')
     parser.add_argument('--analysis-report', action='store_true', default=False, help='Generate analysis report for the evaluation results using judge model.')  # noqa: E501
+
+    # Sandbox-related arguments
+    parser.add_argument('--use-sandbox', action='store_true', default=False, help='Whether to use sandbox for model evaluation.')  # noqa: E501
+    parser.add_argument('--sandbox-type', type=str, default='docker', help='The sandbox type to use.')  # noqa: E501
+    parser.add_argument('--sandbox-config', type=json.loads, default='{}', help='The sandbox config, should be a json string.')  # noqa: E501
+    parser.add_argument('--sandbox-manager-config', type=json.loads, default='{}', help='The sandbox manager config, should be a json string.')  # noqa: E501
     # yapf: enable
 
 

evalscope/backend/opencompass/backend_manager.py

@@ -47,7 +47,6 @@ class OpenCompassBackendManager(BackendManager):
                     datasets: list, the datasets.
                     models: list, the models.
                     work_dir (Optional): str, the working directory. Default to None, which means the current directory.
-                    dry_run (Optional): bool, the dry-run flag. Default to False.
                     debug (Optional): bool, the debug flag. Default to False.
                     reuse (Optional): str, reuse previous outputs & results. Default to None.
                     generation_kwargs (Optional): dict, the generation config. Default to {}.
@@ -140,7 +139,6 @@ class OpenCompassBackendManager(BackendManager):
             cmd_str = f'python -m run_oc ' \
                       f'--models {" ".join(self.args.models)} ' \
                       f'--datasets {" ".join(self.args.datasets)} ' \
-                      f'{self.get_restore_arg("dry-run", self.args.dry_run)} ' \
                       f'{self.get_arg_with_default("work-dir", self.args.work_dir)}'
 
         elif cmd_mode == CmdMode.SCRIPT:

evalscope/backend/rag_eval/utils/embedding.py

@@ -164,6 +164,13 @@ class CrossEncoderModel(BaseModel):
             max_length=self.max_seq_length,
             automodel_args=self.model_kwargs,
         )
+        self.tokenizer = self.model.tokenizer
+        # set pad token
+        if self.tokenizer.pad_token is None:
+            self.tokenizer.pad_token = self.tokenizer.eos_token
+        if ('pad_token_id' not in self.model.config) or (self.model.config.pad_token_id is None):
+            self.model.config.update({'pad_token_id': self.tokenizer.eos_token_id})
+
         self.supported_encode_params = get_supported_params(self.model.predict)
 
     def predict(self, sentences: List[List[str]], **kwargs) -> Tensor:
@@ -189,6 +196,7 @@ class APIEmbeddingModel(BaseModel):
         self.openai_api_base = kwargs.get('api_base')
         self.openai_api_key = kwargs.get('api_key')
         self.dimensions = kwargs.get('dimensions')
+        self.check_embedding_ctx_length = kwargs.get('check_embedding_ctx_length', False)
         self.framework = ['API']
 
         self.model = OpenAIEmbeddings(
@@ -196,7 +204,7 @@ class APIEmbeddingModel(BaseModel):
             openai_api_base=self.openai_api_base,
             openai_api_key=self.openai_api_key,
             dimensions=self.dimensions,
-            check_embedding_ctx_length=False
+            check_embedding_ctx_length=self.check_embedding_ctx_length,
         )
 
         super().__init__(model_name_or_path=self.model_name, **kwargs)

evalscope/benchmarks/ai2d/ai2d_adapter.py

@@ -0,0 +1,53 @@
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, parse_answers, prompt
+
+logger = get_logger()
+
+SUBSET_LIST = ['default']
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ai2d',
+        pretty_name='AI2D',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description='A Diagram Is Worth A Dozen Images',
+        dataset_id='lmms-lab/ai2d',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class Ai2dAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        answers_list: list[str] = record['options']
+        input_text = prompt(question=record['question'], choices=answers_list, template=MULT_CHOICE_PROMPT)
+        content_list: list[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+
+        label_answer = chr(int(record['answer']) + ord('A'))
+
+        return Sample(input=[ChatMessageUser(content=content_list)], choices=answers_list, target=label_answer)
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        answers = parse_answers(task_state)
+        return ''.join(sorted(list(answers)))

evalscope/benchmarks/amc/amc_adapter.py

@@ -0,0 +1,46 @@
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='amc',
+        pretty_name='AMC',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'AMC (American Mathematics Competitions) is a series of mathematics competitions for high school students.',
+        dataset_id='evalscope/amc_22-24',
+        subset_list=['amc22', 'amc23', 'amc24'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        prompt_template='{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+    )
+)
+class AMCAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # Use split as subset
+        self.split_as_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=record['problem'],
+            target=record['answer'],
+            metadata={
+                'year': record['year'],
+                'url': record['url'],
+                'solution': record.get('solution', '')
+            },
+        )

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -141,35 +141,61 @@ class BBHAdapter(DefaultDataAdapter):
     @classmethod
     def _extract_mc_answer(cls, ans: str) -> str:
         """
-        Extract the answer from the model output for Multiple choice task.
+        Extract normalized answer for BBH multiple-choice tasks.
+        Handles formats like:
+        - "answer is (A)"
+        - "The answer is A."
+        - Extra text after answer.
+        Always uses the *last* occurrence of "answer is".
         """
-        ans_line = ans.split('answer is ')
-        if len(ans_line) != 1:
-            ans = ans_line[1].strip()
-        match = re.search(r'\(([A-Z])\)*', ans)
+        ans = ans.strip()
+
+        parts = ans.split('So the answer is ')
+        if len(parts) > 1:
+            ans = parts[-1].strip()
+        ans = ans.split('\n')[0].strip()
+
+        # Remove trailing period
+        if ans.endswith('.'):
+            ans = ans[:-1].strip()
+
+        # Capture uppercase letter inside parentheses (A) (B) ...
+        match = re.search(r'\(([A-Z])\)', ans)
         if match:
             return match.group(1)
-        match = re.search(r'([A-Z])', ans)
+
+        # Capture single uppercase letter
+        match = re.search(r'\b([A-Z])\b', ans)
         if match:
             return match.group(1)
+
         return ans
 
     @classmethod
     def _extract_ff_answer(cls, ans: str):
         """
-        Extract the answer from the model output for Free-form task.
+        Extract the normalized answer for BBH free-form tasks.
+        Handles patterns like:
+        - "answer is XXX."
+        - "The answer is **valid**."
+        - Extra trailing dots / line breaks.
+        - Bold-marked answers (**xxx**).
+        Always uses the *last* occurrence of "answer is".
         """
-        pattern = r'answer is\s+(.*?)\.'
+        ans = ans.strip()
 
-        match = re.search(pattern, ans)
-        if match:
-            res = match.group(1)
-            return res
+        parts = ans.split('So the answer is ')
+        if len(parts) > 1:
+            ans = parts[-1].strip()
+        ans = ans.split('\n')[0].strip()
 
-        ans_line = ans.split('answer is ')
-        if len(ans_line) != 1:
-            ans = ans_line[1].strip()
-        ans = ans.split('\n')[0]
+        # Remove trailing period
         if ans.endswith('.'):
-            ans = ans[:-1]
+            ans = ans[:-1].strip()
+
+        # If answer is in bold (**xxx**), prefer the content inside
+        match = re.search(r'\*\*(.*?)\*\*', ans)
+        if match:
+            ans = match.group(1).strip()
+
         return ans