evalscope 1.0.1__py3-none-any.whl → 1.0.2__py3-none-any.whl

evalscope/api/benchmark/adapters/default_data_adapter.py

@@ -642,9 +642,7 @@ class DefaultDataAdapter(DataAdapter):
         """
         pass
 
-    def _on_generate_report(
-        self, scores: Dict[str, List[AggScore]], model_name: str, add_aggregation_name: bool = True
-    ) -> Report:
+    def _on_generate_report(self, scores: Dict[str, List[AggScore]], model_name: str) -> Report:
         """
         Hook method called during report generation.
 
@@ -660,7 +658,7 @@ class DefaultDataAdapter(DataAdapter):
             Report: The generated evaluation report
         """
         return ReportGenerator.generate_report(
-            score_dict=scores, model_name=model_name, data_adapter=self, add_aggregation_name=add_aggregation_name
+            score_dict=scores, model_name=model_name, data_adapter=self, add_aggregation_name=self.add_aggregation_name
         )
 
     @override
@@ -682,3 +680,7 @@ class DefaultDataAdapter(DataAdapter):
         report = self._on_generate_report(scores, model_name=model_name)
         self._on_generate_report_end(report, output_dir, **kwargs)
         return report
+
+    def finalize(self, *args, **kwargs):
+        # Finalize the evaluation process
+        self.sandbox_finalize(*args, **kwargs)

evalscope/api/benchmark/adapters/multi_choice_adapter.py

@@ -18,8 +18,11 @@ class MultiChoiceAdapter(DefaultDataAdapter):
     This adapter formats the input for multi-choice questions and handles few-shot examples.
     """
 
-    multiple_correct: bool = False
-    """Whether the benchmark allows multiple correct answers."""
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.multiple_correct: bool = False
+        """Whether the benchmark allows multiple correct answers."""
 
     def format_prompt_template(self, sample: Sample) -> str:
         """

evalscope/api/benchmark/adapters/text2image_adapter.py

@@ -19,6 +19,11 @@ logger = get_logger()
 class Text2ImageAdapter(DefaultDataAdapter):
     """Text to Image Adapter for benchmarks."""
 
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.add_aggregation_name = False  # Do not add aggregation name in the report by default
+
     def load_from_disk(self, **kwargs):
         return super().load_from_disk(use_local_loader=True)
 
@@ -150,7 +155,3 @@ class Text2ImageAdapter(DefaultDataAdapter):
                 score.metadata[metric_name] = f'error: {str(e)}'
 
         return score
-
-    def _on_generate_report(self, scores, model_name, add_aggregation_name=True):
-        # Don't add aggregation name for needle haystack adapter
-        return super()._on_generate_report(scores, model_name, False)

evalscope/api/benchmark/adapters/vision_language_adapter.py

@@ -3,4 +3,6 @@ from .default_data_adapter import DefaultDataAdapter
 
 class VisionLanguageAdapter(DefaultDataAdapter):
     """Adapter for vision-language benchmarks. e.g., image captioning, visual question answering, etc."""
-    pass
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)

evalscope/api/benchmark/benchmark.py

@@ -9,7 +9,7 @@ from evalscope.api.dataset import DatasetDict, Sample
 from evalscope.api.evaluator import TaskState
 from evalscope.api.filter import FilterEnsemble, build_filter_ensemble
 from evalscope.api.metric import AggScore, SampleScore
-from evalscope.api.mixin import LLMJudgeMixin
+from evalscope.api.mixin import LLMJudgeMixin, SandboxMixin
 from evalscope.api.model import Model
 from evalscope.report import Report
 from evalscope.utils.logger import get_logger
@@ -21,7 +21,7 @@ if TYPE_CHECKING:
 logger = get_logger()
 
 
-class DataAdapter(LLMJudgeMixin, ABC):
+class DataAdapter(LLMJudgeMixin, SandboxMixin, ABC):
     """
     Data Adapter for the benchmark.
     """
@@ -43,6 +43,12 @@ class DataAdapter(LLMJudgeMixin, ABC):
         self.save_metadata = True
         """Whether to save metadata in the review result"""
 
+        self.add_aggregation_name = True
+        """Whether to add aggregation name in the report"""
+
+        self.add_overall_metric = True
+        """Whether to add overall metric in the report"""
+
         self.category_map = {}
         """Category map for the benchmark"""
 
@@ -86,6 +92,11 @@ class DataAdapter(LLMJudgeMixin, ABC):
         """
         pass
 
+    @abstractmethod
+    def finalize(self, *args, **kwargs) -> None:
+        """Finalize the evaluation process."""
+        pass
+
     @property
     def name(self) -> str:
         """
@@ -334,6 +345,20 @@ class DataAdapter(LLMJudgeMixin, ABC):
         """
         self._benchmark_meta.shuffle_choices = value
 
+    @property
+    def review_timeout(self) -> Optional[float]:
+        """
+        Return the timeout for the review process.
+        """
+        return self._benchmark_meta.review_timeout
+
+    @review_timeout.setter
+    def review_timeout(self, value: float):
+        """
+        Set the timeout for the review process.
+        """
+        self._benchmark_meta.review_timeout = value
+
     @contextlib.contextmanager
     def _temporary_attribute(self, attr_name: str, new_value):
         """

evalscope/api/benchmark/meta.py

@@ -79,6 +79,9 @@ class BenchmarkMeta:
     shuffle_choices: bool = False
     """Whether to shuffle the choices in multiple-choice datasets."""
 
+    review_timeout: Optional[float] = None
+    """ Timeout for review in seconds."""
+
     extra_params: Dict = field(default_factory=dict)
     """ Additional parameters for the benchmark."""
 

evalscope/api/evaluator/evaluator.py

@@ -54,3 +54,8 @@ class Evaluator(abc.ABC):
     def get_report(self, *args, **kwargs) -> Report:
         """Get the evaluation report."""
         pass
+
+    @abc.abstractmethod
+    def finalize(self, *args, **kwargs) -> None:
+        """Finalize the evaluation process."""
+        pass

evalscope/api/evaluator/state.py

@@ -273,3 +273,8 @@ class TaskState:
     def target(self) -> str:
         """The scoring target for this `Sample`."""
         return self._target.text
+
+    @target.setter
+    def target(self, text: str) -> None:
+        """Set the target for review purposes."""
+        self._target = Target(text)

evalscope/api/messages/chat_message.py

@@ -3,7 +3,7 @@ from pydantic import BaseModel, Field, JsonValue, model_validator
 from typing import Any, Dict, List, Literal, Optional, Type, Union
 
 from evalscope.api.tool import ToolCall, ToolCallError
-from .content import Content, ContentImage, ContentReasoning, ContentText
+from .content import Content, ContentAudio, ContentImage, ContentReasoning, ContentText
 from .utils import parse_content_with_reasoning
 
 
@@ -225,6 +225,11 @@ def messages_to_markdown(messages: List[ChatMessage], max_length: Optional[int]
                     if max_length and len(image_base64) > max_length:
                         image_base64 = image_base64[:max_length]
                     content_parts.append(f'![image]({image_base64})')
+                elif isinstance(content_item, ContentAudio):
+                    audio_base64 = content_item.audio
+                    if max_length and len(audio_base64) > max_length:
+                        audio_base64 = audio_base64[:max_length]
+                    content_parts.append(f"<audio controls src='{audio_base64}'></audio>")
                 elif isinstance(content_item, ContentReasoning):
                     content_parts.append(f'**Reasoning:** {content_item.reasoning}')
 

evalscope/api/mixin/__init__.py

@@ -1 +1,2 @@
 from .llm_judge_mixin import LLMJudgeMixin
+from .sandbox_mixin import SandboxMixin

evalscope/api/mixin/llm_judge_mixin.py

@@ -24,6 +24,8 @@ class LLMJudgeMixin:
 
         self._llm_judge: Optional[LLMJudge] = None
 
+        super().__init__(task_config=task_config)
+
     @property
     def llm_judge(self) -> Optional[LLMJudge]:
         """Get LLM judge instance with lazy initialization."""

evalscope/api/mixin/sandbox_mixin.py

@@ -0,0 +1,204 @@
+import asyncio
+import threading
+from typing import TYPE_CHECKING, Any, Dict, List, Optional
+
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from ms_enclave.sandbox.manager import SandboxManager
+
+    from evalscope.config import TaskConfig
+
+logger = get_logger()
+
+
+class SandboxMixin:
+    """Sandbox mixin for sandboxed code execution."""
+
+    def __init__(self, task_config: 'TaskConfig'):
+        self._task_config = task_config
+
+        self._manager: Optional['SandboxManager'] = None
+        """Sandbox manager instance."""
+
+        self._sandbox_id: Optional[str] = None
+        """Sandbox ID."""
+
+        self._loop: Optional[asyncio.AbstractEventLoop] = None
+        """Event loop for async operations."""
+
+        # Initialize sandbox synchronously by running async methods
+        if self.use_sandbox:
+            self._loop = asyncio.new_event_loop()
+
+            # Start the loop in a separate thread
+            def run_loop():
+                asyncio.set_event_loop(self._loop)
+                self._loop.run_forever()
+
+            self._loop_thread = threading.Thread(target=run_loop, daemon=True)
+            self._loop_thread.start()
+
+            # Wait for initialization
+            future = asyncio.run_coroutine_threadsafe(self._async_init(), self._loop)
+            future.result()
+
+        super().__init__()
+
+    async def _async_init(self):
+        """Async initialization helper."""
+        await self.init_sandbox_manager_async()
+        await self.init_sandbox_async()
+
+    @property
+    def use_sandbox(self) -> bool:
+        """
+        Return whether to use sandbox for the benchmark.
+        """
+        if not self._task_config:
+            return False
+        else:
+            return self._task_config.use_sandbox
+
+    @property
+    def sandbox_manager(self) -> Optional['SandboxManager']:
+        """Get the sandbox manager instance."""
+        return self._manager
+
+    @property
+    def sandbox_id(self) -> Optional[str]:
+        """Get the sandbox ID."""
+        return self._sandbox_id
+
+    async def init_sandbox_manager_async(self) -> Optional['SandboxManager']:
+        """Initialize the sandbox manager asynchronously."""
+        if self._manager is not None:
+            return self._manager
+
+        if not self.use_sandbox:
+            return None
+
+        from ms_enclave.sandbox.manager import HttpSandboxManager, LocalSandboxManager
+
+        manager_config = self._task_config.sandbox_manager_config or {}
+        if manager_config.get('base_url'):
+            # Remote manager
+            self._manager = HttpSandboxManager(**manager_config)
+        else:
+            # Local manager
+            self._manager = LocalSandboxManager(**manager_config)
+
+        await self._manager.start()
+        logger.info('Sandbox manager initialized.')
+        return self._manager
+
+    def init_sandbox_manager(self) -> Optional['SandboxManager']:
+        """Initialize the sandbox manager."""
+        if self._manager is not None:
+            return self._manager
+
+        if not self.use_sandbox:
+            return None
+
+        # Use the dedicated loop if available
+        if self._loop and not self._loop.is_closed():
+            future = asyncio.run_coroutine_threadsafe(self.init_sandbox_manager_async(), self._loop)
+            return future.result()
+        else:
+            # Fallback for cases where no loop is available
+            return asyncio.run(self.init_sandbox_manager_async())
+
+    async def init_sandbox_async(self) -> Optional[str]:
+        """Initialize the sandbox instance asynchronously."""
+        if self._sandbox_id is not None:
+            return self._sandbox_id
+
+        if not self.use_sandbox:
+            return None
+
+        from ms_enclave.sandbox.model import DockerSandboxConfig, SandboxType
+
+        sandbox_config = self._task_config.sandbox_config or DockerSandboxConfig(
+            image='python:3.11-slim', tools_config={
+                'shell_executor': {},
+                'python_executor': {}
+            }
+        )
+        sandbox_type = self._task_config.sandbox_type or SandboxType.DOCKER
+
+        self._sandbox_id = await self._manager.create_sandbox(sandbox_type=sandbox_type, config=sandbox_config)
+
+        sandbox_info = await self._manager.get_sandbox_info(self._sandbox_id)
+
+        logger.info(f'Sandbox of type {sandbox_type} initialized. Info: {sandbox_info.model_dump(exclude_none=True)}')
+        return self._sandbox_id
+
+    def init_sandbox(self) -> Optional[str]:
+        """Initialize the sandbox instance."""
+        if self._sandbox_id is not None:
+            return self._sandbox_id
+
+        if not self.use_sandbox:
+            return None
+
+        # Use the dedicated loop if available
+        if self._loop and not self._loop.is_closed():
+            future = asyncio.run_coroutine_threadsafe(self.init_sandbox_async(), self._loop)
+            return future.result()
+        else:
+            # Fallback for cases where no loop is available
+            return asyncio.run(self.init_sandbox_async())
+
+    def execute_code_in_sandbox(self, code: str, timeout: int = 60, language: str = 'python') -> Dict[str, Any]:
+        """Execute code in the sandbox."""
+        if not self._sandbox_id or not self._manager:
+            logger.warning('Sandbox is not initialized.')
+            return {'error': 'Sandbox is not initialized.'}
+
+        from ms_enclave.sandbox.model import ExecutionStatus, ToolResult
+
+        async def _execute_async():
+            if language.lower() == 'python':
+                tool_name = 'python_executor'
+                parameters = {'code': code, 'timeout': timeout}
+                result = await self._manager.execute_tool(self._sandbox_id, tool_name, parameters)
+            elif language.lower() == 'shell':
+                tool_name = 'shell_executor'
+                parameters = {'command': code, 'timeout': timeout}
+                result = await self._manager.execute_tool(self._sandbox_id, tool_name, parameters)
+            else:
+                logger.warning(f"Unsupported language: {language}. Supported languages are 'python' and 'shell'.")
+                result = ToolResult(
+                    status=ExecutionStatus.ERROR,
+                    tool_name='code_executor',
+                    output=f"Unsupported language: {language}. Supported languages are 'python' and 'shell'."
+                )
+            return result
+
+        # Use the dedicated loop if available
+        if self._loop and not self._loop.is_closed():
+            future = asyncio.run_coroutine_threadsafe(_execute_async(), self._loop)
+            result = future.result(timeout + 10)  # Add some buffer to the timeout
+        else:
+            # Fallback for cases where no loop is available
+            result = asyncio.run(_execute_async())
+
+        return result.model_dump(exclude_none=True)
+
+    def sandbox_finalize(self, *args, **kwargs):
+        """Finalize the sandbox manager."""
+        if self._manager:
+            try:
+                if self._loop and not self._loop.is_closed():
+                    # Stop the manager using the dedicated loop
+                    future = asyncio.run_coroutine_threadsafe(self._manager.stop(), self._loop)
+                    future.result(timeout=30)
+
+                    # Stop the event loop
+                    self._loop.call_soon_threadsafe(self._loop.stop)
+                    if hasattr(self, '_loop_thread'):
+                        self._loop_thread.join(timeout=5)
+
+                logger.info('Sandbox manager finalized.')
+            except Exception as e:
+                logger.warning(f'Error finalizing sandbox manager: {e}')

evalscope/api/model/generate_config.py

@@ -36,9 +36,6 @@ class GenerateConfig(BaseModel):
     stream: Optional[bool] = Field(default=None)
     """Whether to stream the response (default is model specific)."""
 
-    system_message: Optional[str] = Field(default=None)
-    """Override the default system message."""
-
     max_tokens: Optional[int] = Field(default=None)
     """The maximum number of tokens that can be generated in the completion (default is model specific)."""
 

evalscope/api/model/model.py

@@ -365,7 +365,7 @@ def get_model(
 
     logger.info(
         f'Creating model {model} with eval_type={eval_type} '
-        f'base_url={base_url}, api_key={api_key}, config={config}, model_args={model_args}'
+        f'base_url={base_url}, config={config.model_dump(exclude_none=True)}, model_args={model_args}'
     )
 
     # find a matching model type

evalscope/api/tool/tool_info.py

@@ -1,7 +1,7 @@
 import inspect
 from dataclasses import dataclass
 from docstring_parser import Docstring, parse
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field, field_validator
 from typing import Any, Callable, Dict, List, Literal, Optional, TypeAlias, Union, get_args, get_type_hints
 
 from evalscope.utils.json_schema import JSONSchema, JSONType, json_schema, python_type_to_json_type

evalscope/arguments.py

@@ -87,6 +87,12 @@ def add_argument(parser: argparse.ArgumentParser):
     parser.add_argument('--judge-model-args', type=json.loads, default='{}', help='The judge model args, should be a json string.')  # noqa: E501
     parser.add_argument('--judge-worker-num', type=int, default=1, help='The number of workers for the judge model.')
     parser.add_argument('--analysis-report', action='store_true', default=False, help='Generate analysis report for the evaluation results using judge model.')  # noqa: E501
+
+    # Sandbox-related arguments
+    parser.add_argument('--use-sandbox', action='store_true', default=False, help='Whether to use sandbox for model evaluation.')  # noqa: E501
+    parser.add_argument('--sandbox-type', type=str, default='docker', help='The sandbox type to use.')  # noqa: E501
+    parser.add_argument('--sandbox-config', type=json.loads, default='{}', help='The sandbox config, should be a json string.')  # noqa: E501
+    parser.add_argument('--sandbox-manager-config', type=json.loads, default='{}', help='The sandbox manager config, should be a json string.')  # noqa: E501
     # yapf: enable
 
 

evalscope/benchmarks/ai2d/ai2d_adapter.py

@@ -0,0 +1,53 @@
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+from evalscope.utils.multi_choices import MultipleChoiceTemplate, parse_answers, prompt
+
+logger = get_logger()
+
+SUBSET_LIST = ['default']
+
+MULT_CHOICE_PROMPT = MultipleChoiceTemplate.SINGLE_ANSWER_COT
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='ai2d',
+        pretty_name='AI2D',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description='A Diagram Is Worth A Dozen Images',
+        dataset_id='lmms-lab/ai2d',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template=MULT_CHOICE_PROMPT,
+    )
+)
+class Ai2dAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        answers_list: list[str] = record['options']
+        input_text = prompt(question=record['question'], choices=answers_list, template=MULT_CHOICE_PROMPT)
+        content_list: list[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+
+        label_answer = chr(int(record['answer']) + ord('A'))
+
+        return Sample(input=[ChatMessageUser(content=content_list)], choices=answers_list, target=label_answer)
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        answers = parse_answers(task_state)
+        return ''.join(sorted(list(answers)))

evalscope/benchmarks/amc/amc_adapter.py

@@ -0,0 +1,46 @@
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='amc',
+        pretty_name='AMC',
+        tags=[Tags.MATH, Tags.REASONING],
+        description=
+        'AMC (American Mathematics Competitions) is a series of mathematics competitions for high school students.',
+        dataset_id='evalscope/amc_22-24',
+        subset_list=['amc22', 'amc23', 'amc24'],
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        prompt_template='{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.',
+    )
+)
+class AMCAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        # Use split as subset
+        self.split_as_subset = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        return Sample(
+            input=record['problem'],
+            target=record['answer'],
+            metadata={
+                'year': record['year'],
+                'url': record['url'],
+                'solution': record.get('solution', '')
+            },
+        )

evalscope/benchmarks/bbh/bbh_adapter.py

@@ -141,35 +141,61 @@ class BBHAdapter(DefaultDataAdapter):
     @classmethod
     def _extract_mc_answer(cls, ans: str) -> str:
         """
-        Extract the answer from the model output for Multiple choice task.
+        Extract normalized answer for BBH multiple-choice tasks.
+        Handles formats like:
+        - "answer is (A)"
+        - "The answer is A."
+        - Extra text after answer.
+        Always uses the *last* occurrence of "answer is".
         """
-        ans_line = ans.split('answer is ')
-        if len(ans_line) != 1:
-            ans = ans_line[1].strip()
-        match = re.search(r'\(([A-Z])\)*', ans)
+        ans = ans.strip()
+
+        parts = ans.split('So the answer is ')
+        if len(parts) > 1:
+            ans = parts[-1].strip()
+        ans = ans.split('\n')[0].strip()
+
+        # Remove trailing period
+        if ans.endswith('.'):
+            ans = ans[:-1].strip()
+
+        # Capture uppercase letter inside parentheses (A) (B) ...
+        match = re.search(r'\(([A-Z])\)', ans)
         if match:
             return match.group(1)
-        match = re.search(r'([A-Z])', ans)
+
+        # Capture single uppercase letter
+        match = re.search(r'\b([A-Z])\b', ans)
         if match:
             return match.group(1)
+
         return ans
 
     @classmethod
     def _extract_ff_answer(cls, ans: str):
         """
-        Extract the answer from the model output for Free-form task.
+        Extract the normalized answer for BBH free-form tasks.
+        Handles patterns like:
+        - "answer is XXX."
+        - "The answer is **valid**."
+        - Extra trailing dots / line breaks.
+        - Bold-marked answers (**xxx**).
+        Always uses the *last* occurrence of "answer is".
         """
-        pattern = r'answer is\s+(.*?)\.'
+        ans = ans.strip()
 
-        match = re.search(pattern, ans)
-        if match:
-            res = match.group(1)
-            return res
+        parts = ans.split('So the answer is ')
+        if len(parts) > 1:
+            ans = parts[-1].strip()
+        ans = ans.split('\n')[0].strip()
 
-        ans_line = ans.split('answer is ')
-        if len(ans_line) != 1:
-            ans = ans_line[1].strip()
-        ans = ans.split('\n')[0]
+        # Remove trailing period
         if ans.endswith('.'):
-            ans = ans[:-1]
+            ans = ans[:-1].strip()
+
+        # If answer is in bold (**xxx**), prefer the content inside
+        match = re.search(r'\*\*(.*?)\*\*', ans)
+        if match:
+            ans = match.group(1).strip()
+
         return ans