evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/live_code_bench/live_code_bench_adapter.py

@@ -1,3 +1,4 @@
+# flake8: noqa: E501
 from typing import Any, Dict
 
 from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
@@ -7,7 +8,7 @@ from evalscope.api.messages.chat_message import ChatMessageUser
 from evalscope.api.metric import Score
 from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
-from evalscope.utils.io_utils import convert_numpy_types
+from evalscope.utils.io_utils import convert_normal_types
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
@@ -19,17 +20,18 @@ logger = get_logger()
         pretty_name='Live-Code-Bench',
         tags=[Tags.CODING],
         description=
-        'Live Code Bench is a benchmark for evaluating code generation models on real-world coding tasks. It includes a variety of programming problems with test cases to assess the model\'s ability to generate correct and efficient code solutions.',  # noqa: E501
+        'Live Code Bench is a benchmark for evaluating code generation models on real-world coding tasks. It includes a variety of programming problems with test cases to assess the model\'s ability to generate correct and efficient code solutions. '
+        '**By default the code is executed in local environment. We recommend using sandbox execution to safely run and evaluate the generated code, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html) for more details.**',
         dataset_id='AI-ModelScope/code_generation_lite',
         subset_list=['release_latest'],
-        metric_list=['Pass@1'],
+        aggregation='mean_and_pass_at_k',
         eval_split='test',
         prompt_template=
         '### Question:\n{question_content}\n\n{format_prompt} ### Answer: (use the provided format with backticks)\n\n',
+        review_timeout=6,
         extra_params={
             'start_date': None,
             'end_date': None,
-            'timeout': 6,
             'debug': False
         },
     )
@@ -42,7 +44,6 @@ class LiveCodeBenchAdapter(DefaultDataAdapter):
     def __init__(self, **kwargs):
         super().__init__(**kwargs)
 
-        self.timeout = self.extra_params.get('timeout', 6)
         self.debug = self.extra_params.get('debug', False)
         self.start_date = self.extra_params.get('start_date')
         self.end_date = self.extra_params.get('end_date')
@@ -81,58 +82,69 @@ class LiveCodeBenchAdapter(DefaultDataAdapter):
     def match_score(
         self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
     ) -> Score:
-        from .evaluate_utils import codegen_metrics
-
         score = Score(
             extracted_prediction=filtered_prediction,
             prediction=original_prediction,
         )
 
-        references = [{'input_output': task_state.metadata['evaluation_sample']}]
-        predictions = [[filtered_prediction]]
-
-        try:
-            metrics, eval_results, final_metadata = codegen_metrics(
-                references,
-                predictions,
-                k_list=[1],
-                num_process_evaluate=1,
-                timeout=self.timeout,
-                debug=self.debug,
-            )
-            pass_rate = metrics['pass@1'] / 100  # convert to point scale
-
-            score.value = {'pass': float(pass_rate > 0)}
-            score.explanation = f"Pass@1: {metrics['pass@1']}%"
-
-            # Convert numpy types to native Python types for JSON serialization
-            serializable_eval_results = convert_numpy_types(eval_results)
-            serializable_final_metadata = convert_numpy_types(final_metadata)
-
-            score.metadata = {
-                'pass_rate': float(pass_rate),
-                'timeout': self.timeout,
-                'debug': self.debug,
-                'eval_results': serializable_eval_results,
-                'final_metadata': serializable_final_metadata
-            }
-        except Exception as e:
-            score.value = {'pass': False}
-            score.explanation = f'Evaluation failed: {str(e)}'
-            score.metadata = {'error': str(e)}
-
-        score.main_score_name = 'pass'
+        if not self.use_sandbox:
+            # Use original evaluation method
+            from .evaluate_utils import codegen_metrics
+
+            references = [{'input_output': task_state.metadata['evaluation_sample']}]
+            predictions = [[filtered_prediction]]
+
+            try:
+                metrics, eval_results, final_metadata = codegen_metrics(
+                    references,
+                    predictions,
+                    k_list=[1],
+                    num_process_evaluate=1,
+                    timeout=self.review_timeout,
+                    debug=self.debug,
+                )
+                pass_rate = metrics['pass@1'] / 100  # convert to point scale
+
+                score.value = {'acc': float(pass_rate > 0)}
+                score.explanation = f"Pass@1: {metrics['pass@1']}%"
+
+                # Convert numpy types to native Python types for JSON serialization
+                serializable_eval_results = convert_normal_types(eval_results)
+                serializable_final_metadata = convert_normal_types(final_metadata)
+
+                score.metadata = {
+                    'pass_rate': float(pass_rate),
+                    'timeout': self.review_timeout,
+                    'debug': self.debug,
+                    'eval_results': serializable_eval_results,
+                    'final_metadata': serializable_final_metadata
+                }
+            except Exception as e:
+                score.value = {'acc': False}
+                score.explanation = f'Evaluation failed: {str(e)}'
+                score.metadata = {'error': str(e)}
+        else:
+            # Use sandbox execution
+            try:
+                from .sandbox_evaluate_utils import evaluate_in_sandbox
+
+                evaluation_sample = task_state.metadata['evaluation_sample']
+                passed, detailed_results = evaluate_in_sandbox(
+                    self, filtered_prediction, evaluation_sample, timeout=self.review_timeout, debug=self.debug
+                )
+
+                score.value = {'acc': passed}
+                score.explanation = f"Sandbox execution: {'Passed' if passed else 'Failed'}"
+                score.metadata = {
+                    'timeout': self.review_timeout,
+                    'debug': self.debug,
+                    'execution_method': 'sandbox',
+                    'detailed_results': detailed_results
+                }
+            except Exception as e:
+                score.value = {'acc': False}
+                score.explanation = f'Sandbox evaluation failed: {str(e)}'
+                score.metadata = {'error': str(e), 'execution_method': 'sandbox'}
+
+        score.main_score_name = 'acc'
         return score
-
-    def aggregate_scores(self, sample_scores):
-        from evalscope.metrics.metric import PassAtK
-
-        # calculate pass@k here
-        agg_list = []
-        for metric in self.metric_list:
-            if metric.lower().startswith('pass@'):
-                k = int(metric.split('@')[1])
-                # Get the scores for this metric
-                agg = PassAtK(k)
-                agg_list.extend(agg(sample_scores))
-        return agg_list

evalscope/benchmarks/live_code_bench/sandbox_evaluate_utils.py

@@ -0,0 +1,220 @@
+import json
+from typing import TYPE_CHECKING, Dict, List, Tuple
+
+from evalscope.utils.logger import get_logger
+
+if TYPE_CHECKING:
+    from evalscope.api.mixin.sandbox_mixin import SandboxMixin
+
+logger = get_logger()
+
+
+def evaluate_in_sandbox(
+    adapter: 'SandboxMixin',
+    code: str,
+    evaluation_sample: str,
+    timeout: int = 6,
+    debug: bool = False
+) -> Tuple[bool, Dict]:
+    """
+    Evaluate code in sandbox environment for Live Code Bench.
+
+    Args:
+        adapter: The adapter instance with sandbox capabilities
+        code: The code to evaluate
+        evaluation_sample: JSON string containing input/output test cases
+        timeout: Timeout for execution
+        debug: Whether to enable debug logging
+
+    Returns:
+        Tuple[bool, Dict]: (overall_pass, detailed_results)
+    """
+    try:
+        # Parse the evaluation sample
+        test_data = json.loads(evaluation_sample)
+        inputs = test_data.get('inputs', [])
+        outputs = test_data.get('outputs', [])
+        fn_name = test_data.get('fn_name')
+
+        if debug:
+            logger.info(f'Evaluating code with {len(inputs)} test cases')
+            logger.info(f'Function name: {fn_name}')
+
+        # Determine if this is call-based or stdio-based
+        if fn_name:
+            # Call-based evaluation
+            return _evaluate_call_based_in_sandbox(adapter, code, inputs, outputs, fn_name, timeout, debug)
+        else:
+            # Standard input/output evaluation
+            return _evaluate_stdio_in_sandbox(adapter, code, inputs, outputs, timeout, debug)
+
+    except Exception as e:
+        if debug:
+            logger.error(f'Sandbox evaluation error: {str(e)}')
+        return False, {'error': str(e), 'total_tests': 0, 'passed_tests': 0}
+
+
+def _evaluate_call_based_in_sandbox(
+    adapter: 'SandboxMixin', code: str, inputs: list, outputs: list, fn_name: str, timeout: int, debug: bool
+) -> Tuple[bool, Dict]:
+    """Evaluate call-based problems in sandbox."""
+    try:
+        all_passed = True
+        passed_count = 0
+        failed_cases = []
+
+        for i, (test_input, expected_output) in enumerate(zip(inputs, outputs)):
+            # Prepare individual test code for each test case
+            test_code = f"""
+import json
+import sys
+
+# User's code
+{code}
+
+# Test execution for single test case
+try:
+    test_input = {repr(test_input)}
+    expected_output = {repr(expected_output)}
+
+    if 'class Solution' in '''{code}''':
+        # LeetCode style
+        solution = Solution()
+        method = getattr(solution, '{fn_name}')
+    else:
+        # Function is directly available
+        method = {fn_name}
+
+    # Parse input if it's JSON string
+    if isinstance(test_input, str):
+        try:
+            test_input = json.loads(test_input)
+        except:
+            pass  # Keep as string if not valid JSON
+
+    # Call the method
+    if isinstance(test_input, list):
+        result = method(*test_input)
+    else:
+        result = method(test_input)
+
+    # Parse expected output if it's JSON string
+    if isinstance(expected_output, str):
+        try:
+            expected_output = json.loads(expected_output)
+        except:
+            pass  # Keep as string if not valid JSON
+
+    # Convert tuple to list for comparison
+    if isinstance(result, tuple):
+        result = list(result)
+
+    if result == expected_output:
+        print("TEST_PASSED")
+    else:
+        print(f"TEST_FAILED: expected {{expected_output}}, got {{result}}")
+
+except Exception as e:
+    print(f"EXECUTION_ERROR: {{str(e)}}")
+    import traceback
+    traceback.print_exc()
+"""
+
+            # Execute in sandbox
+            result = adapter.execute_code_in_sandbox(code=test_code, timeout=timeout, language='python')
+
+            if debug:
+                logger.info(f'Test case {i} execution result: {result}')
+
+            # Check if execution was successful and test passed
+            if result.get('status') == 'success':
+                output = result.get('output', '')
+                if 'TEST_PASSED' in output:
+                    passed_count += 1
+                elif 'TEST_FAILED:' in output:
+                    # Extract failure details from output
+                    for line in output.split('\n'):
+                        if line.startswith('TEST_FAILED:'):
+                            failed_cases.append(f"Test {i}: {line.replace('TEST_FAILED: ', '')}")
+                            break
+                    all_passed = False
+                    break
+                elif 'EXECUTION_ERROR:' in output:
+                    # Extract error details
+                    for line in output.split('\n'):
+                        if line.startswith('EXECUTION_ERROR:'):
+                            failed_cases.append(f'Test {i}: {line}')
+                            break
+                    all_passed = False
+                    break
+                else:
+                    failed_cases.append(f'Test {i}: Unknown error in output. Result: {result}')
+                    all_passed = False
+                    break
+            else:
+                failed_cases.append(f'Test {i}: Sandbox execution failed - Result: {result}')
+                all_passed = False
+                break
+
+        detailed_results = {'total_tests': len(inputs), 'passed_tests': passed_count, 'failed_cases': failed_cases}
+
+        return all_passed, detailed_results
+
+    except Exception as e:
+        if debug:
+            logger.error(f'Call-based evaluation error: {str(e)}')
+        return False, {'error': str(e), 'total_tests': len(inputs), 'passed_tests': 0}
+
+
+def _evaluate_stdio_in_sandbox(
+    adapter: 'SandboxMixin', code: str, inputs: list, outputs: list, timeout: int, debug: bool
+) -> Tuple[bool, Dict]:
+    """Evaluate stdio-based problems in sandbox."""
+    try:
+        all_passed = True
+        passed_count = 0
+        failed_cases = []
+
+        for i, (test_input, expected_output) in enumerate(zip(inputs, outputs)):
+            test_code = f"""
+import sys
+from io import StringIO
+
+# Redirect stdin
+sys.stdin = StringIO('''{test_input}''')
+
+# User's code
+{code}
+"""
+
+            # Execute in sandbox
+            result = adapter.execute_code_in_sandbox(code=test_code, timeout=timeout, language='python')
+
+            if result.get('status') != 'success':
+                if debug:
+                    logger.error(f'Test case {i} execution failed: {result}')
+                failed_cases.append(f'Test {i}: Execution error - Result: {result}')
+                all_passed = False
+                break
+
+            # Compare output
+            actual_output = result.get('output', '').strip()
+            expected_output = expected_output.strip()
+
+            if actual_output == expected_output:
+                passed_count += 1
+            else:
+                if debug:
+                    logger.info(f"Test case {i} failed: expected '{expected_output}', got '{actual_output}'")
+                failed_cases.append(f"Test {i}: Expected '{expected_output}', got '{actual_output}'")
+                all_passed = False
+                break
+
+        detailed_results = {'total_tests': len(inputs), 'passed_tests': passed_count, 'failed_cases': failed_cases}
+
+        return all_passed, detailed_results
+
+    except Exception as e:
+        if debug:
+            logger.error(f'Stdio evaluation error: {str(e)}')
+        return False, {'error': str(e), 'total_tests': len(inputs), 'passed_tests': 0}

evalscope/benchmarks/logi_qa/logi_qa_adapter.py

@@ -0,0 +1,41 @@
+# flake8: noqa: E501
+
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+
+DESCRIPTION = 'LogiQA is a dataset sourced from expert-written questions for testing human Logical reasoning.'
+
+PROMPT_TEMPLATE = r"""
+Answer the following multiple choice question. The entire content of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of {letters}.
+
+{question}
+
+{choices}
+""".strip()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='logi_qa',
+        pretty_name='LogiQA',
+        tags=[Tags.REASONING, Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/logiqa',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split='validation',
+        eval_split='test',
+        prompt_template=PROMPT_TEMPLATE,
+    )
+)
+class LogiQAAdapter(MultiChoiceAdapter):
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=f"{record['context']}\n{record['question']}",
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={},
+        )

evalscope/benchmarks/math_500/math_500_adapter.py

@@ -4,7 +4,6 @@ from typing import Any, Dict
 
 from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
 from evalscope.api.dataset import Sample
-from evalscope.api.evaluator import TaskState
 from evalscope.api.registry import register_benchmark
 from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
@@ -49,3 +48,8 @@ class Math500Adapter(DefaultDataAdapter):
                 'solution': record['solution'],
             },
         )
+
+    def extract_answer(self, prediction: str, task_state):
+        from evalscope.metrics.math_parser import extract_answer
+
+        return extract_answer(prediction)

evalscope/benchmarks/math_qa/math_qa_adapter.py

@@ -0,0 +1,35 @@
+from evalscope.api.benchmark import BenchmarkMeta, MultiChoiceAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.multi_choices import MultipleChoiceTemplate
+
+DESCRIPTION = (
+    'MathQA dataset is gathered by using a new representation language to annotate over the '
+    'AQuA-RAT dataset with fully-specified operational programs.'
+)
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='math_qa',
+        pretty_name='MathQA',
+        tags=[Tags.REASONING, Tags.MATH, Tags.MULTIPLE_CHOICE],
+        description=DESCRIPTION.strip(),
+        dataset_id='extraordinarylab/math-qa',
+        metric_list=['acc'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='test',
+        prompt_template=MultipleChoiceTemplate.SINGLE_ANSWER_COT,
+    )
+)
+class MathQAAdapter(MultiChoiceAdapter):
+
+    def record_to_sample(self, record) -> Sample:
+        return Sample(
+            input=record['question'],
+            choices=record['choices'],
+            target=record['answer'],
+            metadata={'reasoning': record['reasoning']},
+        )

evalscope/benchmarks/math_verse/math_verse_adapter.py

@@ -0,0 +1,105 @@
+# flake8: noqa: E501
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+MULTI_CHOICE_TYPE = 'multi-choice'
+OPEN_TYPE = 'free-form'
+
+OPEN_PROMPT = '{question}\nPlease reason step by step, and put your final answer within \\boxed{{}}.'
+
+MULT_CHOICE_PROMPT = """
+Answer the following multiple choice question. The last line of your response should be of the following format: 'ANSWER: $LETTER' (without quotes) where LETTER is one of A, B, C, D. Think step by step before answering.
+
+{question}
+"""
+
+SUBSET_LIST = ['Text Dominant', 'Text Lite', 'Vision Intensive', 'Vision Dominant', 'Vision Only']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='math_verse',
+        pretty_name='MathVerse',
+        dataset_id='evalscope/MathVerse',
+        tags=[Tags.MATH, Tags.REASONING, Tags.MULTIPLE_CHOICE, Tags.MULTI_MODAL],
+        description=
+        'MathVerse, an all-around visual math benchmark designed for an equitable and in-depth evaluation of MLLMs. 2,612 high-quality, multi-subject math problems with diagrams from publicly available sources. Each problem is then transformed by human annotators into six distinct versions, each offering varying degrees of information content in multi-modality, contributing to 15K test samples in total. This approach allows MathVerse to comprehensively assess whether and how much MLLMs can truly understand the visual diagrams for mathematical reasoning.',
+        subset_list=SUBSET_LIST,
+        metric_list=[{
+            'acc': {
+                'numeric': True
+            }
+        }],
+        default_subset='testmini',
+        eval_split='testmini',
+        prompt_template=OPEN_PROMPT,
+    )
+)
+class MathVerseAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.reformat_subset = True
+        self._use_llm_judge = True
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """
+        Convert a dataset record to a Sample. Unifies handling for both multi-choice and free-form.
+        Builds the content list inline and appends image content if provided.
+
+        Args:
+            record: Raw dataset record.
+
+        Returns:
+            Sample: The standardized sample ready for evaluation.
+        """
+        question_type = record.get('question_type', OPEN_TYPE)
+        question: str = record.get('question', '')
+        content_list: list[Content] = []
+
+        # Choose prompt text based on type; keep a single unified flow for creating Sample
+        if question_type == MULTI_CHOICE_TYPE:
+            prompt_text = MULT_CHOICE_PROMPT.format(question=question).strip()
+        else:
+            prompt_text = OPEN_PROMPT.format(question=question).strip()
+
+        content_list.append(ContentText(text=prompt_text))
+
+        # Append image if exists
+        image = record.get('image')
+        if image and isinstance(image, dict):
+            image_bytes = image.get('bytes')
+            if image_bytes:
+                image_base64 = bytes_to_base64(image_bytes, format='png', add_header=True)
+                content_list.append(ContentImage(image=image_base64))
+
+        metadata: Dict[str, Any] = {
+            'sample_index': record.get('sample_index'),
+            'problem_index': record.get('problem_index'),
+            'problem_version': record.get('problem_version'),
+            'question_type': question_type,
+            'query_wo': record.get('query_wo'),
+            'query_cot': record.get('query_cot'),
+            'question_for_eval': record.get('question_for_eval'),
+        }
+
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=record['answer'],
+            subset_key=record['problem_version'],
+            metadata=metadata,
+        )
+
+    def extract_answer(self, prediction: str, task_state):
+        from evalscope.metrics.math_parser import extract_answer
+
+        return extract_answer(prediction)