evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/live_code_bench/evaluate_utils.py

@@ -0,0 +1,195 @@
+import json
+import multiprocessing
+import numpy as np
+from collections import defaultdict
+
+from evalscope.utils.logger import get_logger
+from .pass_k_utils import compute_metrics_from_results
+
+logger = get_logger()
+
+
+def _temp_run(sample, generation, debug, result, metadata_list, timeout):
+    """Runs a test in a separate process to enforce a timeout.
+    This function is defined at the module's top level to ensure it can be
+    pickled by `multiprocessing.Process`. This is a requirement on platforms
+    like macOS (on Apple Silicon) which use the 'spawn' start method, as
+    nested functions are not picklable.
+    """
+    from .testing_util import run_test
+    res, metadata = run_test(sample, test=generation, debug=debug, timeout=timeout)
+    result.append(res)
+    metadata_list.append(metadata)
+
+
+def codegen_check_correctness(sample, generation, timeout, debug=True):
+    """Check correctness of code generation with a global timeout.
+
+    The global timeout is to catch some extreme/rare cases not handled by the
+    timeouts inside `run_test`
+    """
+
+    manager = multiprocessing.Manager()
+    result = manager.list()
+    metadata_list = manager.list()
+    p = multiprocessing.Process(
+        target=_temp_run,
+        args=(sample, generation, debug, result, metadata_list, timeout),
+    )
+    p.start()
+    global_timeout = (timeout + 1) * len(json.loads(sample['input_output'])['inputs'])
+    if debug:
+        logger.info(f'global timeout = {global_timeout}')
+    p.join(timeout=global_timeout)
+    if p.is_alive():
+        p.kill()
+    if not result:
+        in_outs = json.loads(sample['input_output'])
+        # consider that all tests failed
+        result = [[-1 for i in range(len(in_outs['inputs']))]]
+        if debug:
+            logger.info('global timeout occured: alarm went off')
+    return result[0], metadata_list[0]
+
+
+def evaluate_generations_by_problem(problem_generations: list, sample: list, debug: bool, timeout: int):
+    """Evaluate each problem.
+
+    Args:
+        problem_generations:
+        sample:
+        debug:
+        timeout
+    """
+    # problem_generations: list[str] = args[0]
+    # sample = args[1]
+    # debug: bool = args[2]
+    # timeout: int = args[3]
+
+    res = []
+    metadata = []
+    for o_idx, o in enumerate(problem_generations):
+        curr_res = [-2]
+        try:
+            curr_res, curr_metadata = codegen_check_correctness(sample, o, timeout=timeout, debug=debug)
+            if debug:
+                logger.info(f'\nSuccessful compilation of task {o_idx}!')
+            fixed = []
+            for e in curr_res:
+                if isinstance(e, np.ndarray):
+                    e = e.item(0)
+                if isinstance(e, np.bool_):
+                    e = bool(e)
+                fixed.append(e)
+            curr_res = fixed
+            if not np.all(curr_res):
+                if debug:
+                    logger.info(f'Results were not True for all test cases'  # noqa: F541, E501
+                                f' {curr_res=}\n')
+        except Exception as e:
+            if debug:
+                logger.info(f'Compilation failed, test framework exception'  # noqa: F541, E501
+                            f' = {repr(e)}{e}\n')
+            # break
+            curr_metadata = {}
+        finally:
+            assert isinstance(curr_res, list)
+            assert isinstance(curr_metadata, dict)
+            res.append(curr_res)
+            metadata.append(curr_metadata)
+    if debug:
+        for i, r in enumerate(problem_generations):
+            logger.info(f'Sample\n{r}\nResult\n{res[i]}')
+            logger.info('*' * 30 + '\n\n')
+    return res, metadata
+
+
+def evaluate_generations(
+    samples_list: list,
+    generations_list: list[list[str]],
+    debug: bool = False,
+    num_process_evaluate: int = 16,  # This parameter will be unused
+    timeout=6,
+):
+    """We take the list of code generations and try to compile them and the run
+    their corresponding unit tests which are retrieved from the APPS dataset.
+
+    Args:
+        generations: list of code generations (same order as samples in APPS
+            dataset)
+        level: difficulty level used in the generation, can be "all",
+            "introductory", "interview" or "competition"
+
+    Returns:
+        results: dictionary of results, key is the problem index, value is
+            a list of results for each generation
+        [-2] = compile error, [-1] = runtime error [False] = failed test
+            case [True] = passed test case
+    """
+    results = {}
+    metadata = {}
+
+    for index in range(len(generations_list)):
+        problem_generations = generations_list[index]
+        sample = samples_list[index]
+
+        result, meta = evaluate_generations_by_problem(problem_generations, sample, debug, timeout)
+        results[index] = result
+        metadata[index] = meta
+
+    assert len(results
+               ) == len(generations_list), f'results = {len(results)} inputs = {len(generations_list)} {results=}'
+
+    return results, metadata
+
+
+def codegen_metrics(
+    samples_list,
+    generations_list,
+    k_list=[1, 5, 10, 20, 40, 50, 75, 100, 125, 150, 200, 500, 1000],
+    num_process_evaluate=16,
+    timeout=6,
+    debug=False,
+):
+
+    samples_linear = []
+    generations_linear = []
+    remap_index = []
+    results = defaultdict(list)
+    metadatas = defaultdict(list)
+    for idx, (sample, generation_list) in enumerate(zip(samples_list, generations_list)):
+        assert isinstance(generation_list, list), generations_list[0]
+        for generation in generation_list:
+            assert isinstance(generation, str), generations_list[0]
+            samples_linear.append(sample)
+            generations_linear.append([generation])
+            remap_index.append(idx)
+
+    results_linear, metadatas_linear = evaluate_generations(
+        samples_linear,
+        generations_linear,
+        debug=debug,
+        num_process_evaluate=num_process_evaluate,
+        timeout=timeout,
+    )
+
+    for idx, sub_results in sorted(results_linear.items(), key=lambda x: x[0]):
+        results[remap_index[idx]].append(sub_results[0])
+
+    for idx, sub_metadatas in sorted(metadatas_linear.items(), key=lambda x: x[0]):
+        metadatas[remap_index[idx]].append(sub_metadatas[0])
+
+    metrics = compute_metrics_from_results(results, k_list=k_list)
+
+    final_metadata = []
+    for key in sorted(list(metadatas.keys())):
+        final_metadata.append(metadatas[key])
+    for i in range(len(final_metadata)):
+        if type(final_metadata[i]) is not list:
+            final_metadata[i] = [json.dumps(final_metadata[i])]
+        else:
+            final_metadata[i] = [json.dumps(x) for x in final_metadata[i]]
+
+        assert len(final_metadata[i]) == len(generations_list[0]), f'{len(final_metadata[i])=}'
+
+    return [metrics, results, final_metadata]

evalscope/benchmarks/live_code_bench/extract_utils.py

@@ -0,0 +1,70 @@
+# Copyright LiveCodeBench @ 2024,
+
+import re
+
+
+def extract_code_generation(model_output: str, model_type: str = 'chat'):
+    # modified from
+    outputlines = model_output.split('\n')
+    # TODO: handle codellama
+
+    if model_type == 'base':
+        return model_output.strip()
+    elif model_type == 'chat':
+        indexlines = [i for i, line in enumerate(outputlines) if '```' in line]
+    else:
+        raise ValueError(f'Invalid mode type: {model_type}')
+
+    if len(indexlines) < 2:
+        return ''
+    return '\n'.join(outputlines[indexlines[0] + 1:indexlines[1]])
+
+
+def extract_code_execution(model_output: str, cot: bool = False):
+    pattern = r'\[PYTHON\](.*?)\[\/PYTHON\]'
+    matches = re.findall(pattern, model_output, re.DOTALL)
+    if matches:
+        # fetch the last one
+        model_output = matches[-1]
+
+    if '[PYTHON]' in model_output:
+        model_output
+    if cot:
+        if '[ANSWER]' in model_output:
+            model_output = model_output.split('[ANSWER]')[1].strip()
+    if '==' in model_output:
+        model_output = model_output.split('==')[1].strip()
+    if '[/ANSWER]' in model_output:
+        model_output = model_output.split('[/ANSWER]')[0].strip()
+    else:
+        model_output = model_output.split('\n')[0].strip()
+    return model_output.strip()
+
+
+def extract_test_output_code(model_output: str):
+    outputlines = model_output.split('\n')
+    # find the last line startwith assert...
+    indexlines = [i for i, line in enumerate(outputlines) if line.startswith('assert')]
+    if indexlines:
+        return outputlines[indexlines[-1]]
+
+    # TODO: handle codellama format
+    # if lmstyle and lmstyle == LMStyle.CodeLLaMaInstruct:
+    #     indexlines = \
+    # [i for i, line in enumerate(outputlines) if "PYTHON]" in line]
+    # else:
+
+    # first try to extract ```python if not then try ```
+    indexlines = [i for i, line in enumerate(outputlines) if '```python' in line or '```Python' in line]
+    if indexlines:
+        start_index = indexlines[0]
+    else:
+        start_index = None
+    indexlines = [i for i, line in enumerate(outputlines) if '```' in line]
+    if start_index is not None:
+        indexlines = [i for i in indexlines if i > start_index]
+        indexlines = [start_index] + indexlines
+
+    if len(indexlines) < 2:
+        return ''
+    return '\n'.join(outputlines[indexlines[0] + 1:indexlines[1]])

evalscope/benchmarks/live_code_bench/live_code_bench_adapter.py

@@ -0,0 +1,150 @@
+# flake8: noqa: E501
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import convert_normal_types
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='live_code_bench',
+        pretty_name='Live-Code-Bench',
+        tags=[Tags.CODING],
+        description=
+        'Live Code Bench is a benchmark for evaluating code generation models on real-world coding tasks. It includes a variety of programming problems with test cases to assess the model\'s ability to generate correct and efficient code solutions. '
+        '**By default the code is executed in local environment. We recommend using sandbox execution to safely run and evaluate the generated code, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html) for more details.**',
+        dataset_id='AI-ModelScope/code_generation_lite',
+        subset_list=['release_latest'],
+        aggregation='mean_and_pass_at_k',
+        eval_split='test',
+        prompt_template=
+        '### Question:\n{question_content}\n\n{format_prompt} ### Answer: (use the provided format with backticks)\n\n',
+        review_timeout=6,
+        extra_params={
+            'start_date': None,
+            'end_date': None,
+            'debug': False
+        },
+    )
+)
+class LiveCodeBenchAdapter(DefaultDataAdapter):
+    """
+    Live Code Bench adapter using the new data processing framework.
+    """
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.debug = self.extra_params.get('debug', False)
+        self.start_date = self.extra_params.get('start_date')
+        self.end_date = self.extra_params.get('end_date')
+
+        self.save_metadata = False  # Don't save metadata, since they are large
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        from .load_utils import transform
+
+        record = transform(record)
+
+        question_content = record['question_content']
+        format_prompt = record['format_prompt']
+        full_prompt = self.prompt_template.format(question_content=question_content, format_prompt=format_prompt)
+
+        return Sample(
+            input=[ChatMessageUser(content=full_prompt)],
+            target='',
+            metadata={
+                'evaluation_sample': record['evaluation_sample'],
+                'contest_date': record['contest_date']
+            }
+        )
+
+    def sample_filter(self, sample):
+        from .load_utils import filter_date
+
+        return filter_date(sample.metadata['contest_date'], start_date=self.start_date, end_date=self.end_date)
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        """Extract code from the prediction."""
+        from .extract_utils import extract_code_generation
+        return extract_code_generation(prediction)
+
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        if not self.use_sandbox:
+            # Use original evaluation method
+            from .evaluate_utils import codegen_metrics
+
+            references = [{'input_output': task_state.metadata['evaluation_sample']}]
+            predictions = [[filtered_prediction]]
+
+            try:
+                metrics, eval_results, final_metadata = codegen_metrics(
+                    references,
+                    predictions,
+                    k_list=[1],
+                    num_process_evaluate=1,
+                    timeout=self.review_timeout,
+                    debug=self.debug,
+                )
+                pass_rate = metrics['pass@1'] / 100  # convert to point scale
+
+                score.value = {'acc': float(pass_rate > 0)}
+                score.explanation = f"Pass@1: {metrics['pass@1']}%"
+
+                # Convert numpy types to native Python types for JSON serialization
+                serializable_eval_results = convert_normal_types(eval_results)
+                serializable_final_metadata = convert_normal_types(final_metadata)
+
+                score.metadata = {
+                    'pass_rate': float(pass_rate),
+                    'timeout': self.review_timeout,
+                    'debug': self.debug,
+                    'eval_results': serializable_eval_results,
+                    'final_metadata': serializable_final_metadata
+                }
+            except Exception as e:
+                score.value = {'acc': False}
+                score.explanation = f'Evaluation failed: {str(e)}'
+                score.metadata = {'error': str(e)}
+        else:
+            # Use sandbox execution
+            try:
+                from .sandbox_evaluate_utils import evaluate_in_sandbox
+
+                evaluation_sample = task_state.metadata['evaluation_sample']
+                passed, detailed_results = evaluate_in_sandbox(
+                    self, filtered_prediction, evaluation_sample, timeout=self.review_timeout, debug=self.debug
+                )
+
+                score.value = {'acc': passed}
+                score.explanation = f"Sandbox execution: {'Passed' if passed else 'Failed'}"
+                score.metadata = {
+                    'timeout': self.review_timeout,
+                    'debug': self.debug,
+                    'execution_method': 'sandbox',
+                    'detailed_results': detailed_results
+                }
+            except Exception as e:
+                score.value = {'acc': False}
+                score.explanation = f'Sandbox evaluation failed: {str(e)}'
+                score.metadata = {'error': str(e), 'execution_method': 'sandbox'}
+
+        score.main_score_name = 'acc'
+        return score

evalscope/benchmarks/live_code_bench/load_utils.py

@@ -0,0 +1,63 @@
+import base64
+import json
+import pickle
+import zlib
+from datetime import datetime
+
+from evalscope.benchmarks.live_code_bench.prompts import CodeGenerationPromptConstants
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def transform(item):
+    # Define the dataitem mapping logic
+
+    # starter_code
+    if item['starter_code']:
+        format_prompt = f'### Format: {CodeGenerationPromptConstants.FORMATTING_MESSAGE_WITH_STARTER_CODE}\n'  # noqa: E501
+        format_prompt += f"```python\n{item['starter_code']}\n```\n\n"
+    else:
+        format_prompt = f'### Format: {CodeGenerationPromptConstants.FORMATTING_WITHOUT_STARTER_CODE}\n'  # noqa: E501
+        format_prompt += '```python\n# YOUR CODE HERE\n```\n\n'
+
+    item['format_prompt'] = format_prompt
+
+    # load test cases
+    public_test_cases = item['public_test_cases']
+    public_test_cases = json.loads(item['public_test_cases'])
+
+    private_test_cases = item['private_test_cases']
+    try:
+        private_test_cases = json.loads(item['private_test_cases'])
+    except Exception as e:  # noqa: F841
+        private_test_cases = json.loads(
+            pickle.loads(zlib.decompress(base64.b64decode(private_test_cases.encode('utf-8'))))
+        )
+
+    # load metadata
+    metadata = json.loads(item['metadata'])
+    evaluation_sample = json.dumps({
+        'inputs': [t['input'] for t in public_test_cases + private_test_cases],
+        'outputs': [t['output'] for t in public_test_cases + private_test_cases],
+        'fn_name': metadata.get('func_name', None),
+    })
+    item['evaluation_sample'] = evaluation_sample
+
+    return item
+
+
+def filter_date(contest_date, start_date=None, end_date=None) -> bool:
+
+    contest_date = datetime.fromisoformat(contest_date)
+    if start_date is not None:
+        p_start_date = datetime.strptime(start_date, '%Y-%m-%d')
+        if p_start_date > contest_date:
+            return False
+
+    if end_date is not None:
+        p_end_date = datetime.strptime(end_date, '%Y-%m-%d')
+        if p_end_date < contest_date:
+            return False
+
+    return True

evalscope/benchmarks/live_code_bench/pass_k_utils.py

@@ -0,0 +1,56 @@
+# Copyright LiveCodeBench @ 2024,
+
+import numpy as np
+
+
+def estimate_pass_at_k(num_samples, num_correct, k):
+    """Estimates pass@k of each problem and returns them in an array."""
+
+    def estimator(n: int, c: int, k: int) -> float:
+        """Calculates 1 - comb(n - c, k) / comb(n, k)."""
+        if n - c < k:
+            return 1.0 * 100
+        return 100 * (1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1)))
+
+    import itertools
+
+    if isinstance(num_samples, int):
+        num_samples_it = itertools.repeat(num_samples, len(num_correct))
+    else:
+        assert len(num_samples) == len(num_correct)
+        num_samples_it = iter(num_samples)
+
+    return np.array([estimator(int(n), int(c), k) for n, c in zip(num_samples_it, num_correct)])
+
+
+def compute_metrics_from_results(results, k_list=[1, 5]):
+    total = []
+    correct = []
+    task_ids = []
+    for task_id, res in results.items():
+        all_correct = []
+        for generation in res:
+            gen = np.array(generation)
+            all_correct.append(np.all(gen > 0))
+        task_ids.append(task_id)
+        total.append(len(all_correct))
+        correct.append(sum(all_correct))
+    total = np.array(total)
+    correct = np.array(correct)
+    ks = k_list
+    detail_pass_at_k = {f'pass@{k}': estimate_pass_at_k(total, correct, k).tolist() for k in ks if (total >= k).all()}
+    pass_at_k = {f'pass@{k}': estimate_pass_at_k(total, correct, k).mean() for k in ks if (total >= k).all()}
+    detail_metrics = {k: dict(zip(task_ids, v)) for k, v in detail_pass_at_k.items()}
+    pass_at_k['detail'] = detail_metrics
+    return pass_at_k
+
+
+def extract_instance_results(results):
+    instance_wise_grades = {}
+    for task_id, res in results.items():
+        instance_wise_grades[task_id] = []
+        for generation in res:
+            instance_wise_grades[task_id].append(all([g > 0 for g in generation]))
+
+    instance_wise_grades = [v for _, v in sorted(instance_wise_grades.items(), key=lambda item: item[0])]
+    return instance_wise_grades