evalscope 0.12.1__py3-none-any.whl → 0.13.1__py3-none-any.whl

evalscope/benchmarks/live_code_bench/evaluate_utils.py

@@ -0,0 +1,193 @@
+import json
+import multiprocessing
+import numpy as np
+from collections import defaultdict
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+from evalscope.utils.logger import get_logger
+from .pass_k_utils import compute_metrics_from_results
+
+logger = get_logger()
+
+
+def codegen_check_correctness(sample, generation, timeout, debug=True):
+    """Check correctness of code generation with a global timeout.
+
+    The global timeout is to catch some extreme/rare cases not handled by the
+    timeouts inside `run_test`
+    """
+
+    def _temp_run(sample, generation, debug, result, metadata_list, timeout):
+        from .testing_util import run_test
+        res, metadata = run_test(sample, test=generation, debug=debug, timeout=timeout)
+        result.append(res)
+        metadata_list.append(metadata)
+
+    manager = multiprocessing.Manager()
+    result = manager.list()
+    metadata_list = manager.list()
+    p = multiprocessing.Process(
+        target=_temp_run,
+        args=(sample, generation, debug, result, metadata_list, timeout),
+    )
+    p.start()
+    p.join(timeout=(timeout + 1) * len(json.loads(sample['input_output'])['inputs']) + 5)
+    if p.is_alive():
+        p.kill()
+    if not result:
+        in_outs = json.loads(sample['input_output'])
+        # consider that all tests failed
+        result = [[-1 for i in range(len(in_outs['inputs']))]]
+        if debug:
+            logger.info('global timeout')
+    return result[0], metadata_list[0]
+
+
+def evaluate_generations_by_problem(problem_generations: list, sample: list, debug: bool, timeout: int):
+    """Evaluate each problem.
+
+    Args:
+        problem_generations:
+        sample:
+        debug:
+        timeout
+    """
+    # problem_generations: list[str] = args[0]
+    # sample = args[1]
+    # debug: bool = args[2]
+    # timeout: int = args[3]
+
+    res = []
+    metadata = []
+    for o_idx, o in enumerate(problem_generations):
+        curr_res = [-2]
+        try:
+            curr_res, curr_metadata = codegen_check_correctness(sample, o, timeout=timeout, debug=debug)
+            if debug:
+                logger.info(f'\nSuccessful compilation of task {o_idx}!')
+            fixed = []
+            for e in curr_res:
+                if isinstance(e, np.ndarray):
+                    e = e.item(0)
+                if isinstance(e, np.bool_):
+                    e = bool(e)
+                fixed.append(e)
+            curr_res = fixed
+            if not np.all(curr_res):
+                if debug:
+                    logger.info(f'Results were not True for all test cases'  # noqa: F541, E501
+                                f' {curr_res=}\n')
+        except Exception as e:
+            if debug:
+                logger.info(f'Compilation failed, test framework exception'  # noqa: F541, E501
+                            f' = {repr(e)}{e}\n')
+            # break
+            curr_metadata = {}
+        finally:
+            assert isinstance(curr_res, list)
+            assert isinstance(curr_metadata, dict)
+            res.append(curr_res)
+            metadata.append(curr_metadata)
+    if debug:
+        for i, r in enumerate(problem_generations):
+            logger.info(f'Sample\n{r}\nResult\n{res[i]}')
+            logger.info('*' * 30 + '\n\n')
+    return res, metadata
+
+
+def evaluate_generations(
+    samples_list: list,
+    generations_list: list[list[str]],
+    debug: bool = False,
+    num_process_evaluate: int = 16,
+    timeout=6,
+):
+    """We take the list of code generations and try to compile them and the run
+    their corresponding unit tests which are retrieved from the APPS dataset.
+
+    Args:
+        generations: list of code generations (same order as samples in APPS
+            dataset)
+        level: difficulty level used in the generation, can be "all",
+            "introductory", "interview" or "competition"
+
+    Returns:
+        results: dictionary of results, key is the problem index, value is
+            a list of results for each generation
+        [-2] = compile error, [-1] = runtime error [False] = failed test
+            case [True] = passed test case
+    """
+
+    # generations are code generations in the same order of the dataset
+
+    inputs = [[(generations_list[index], samples_list[index], debug, timeout), index]
+              for index in range(len(generations_list))]
+
+    with ProcessPoolExecutor(max_workers=1 if debug else num_process_evaluate) as executor:
+        futures = {
+            executor.submit(evaluate_generations_by_problem, problem_generations, sample, debug, timeout): index
+            for (problem_generations, sample, debug, timeout), index in inputs
+        }
+
+        results = {}
+        metadata = {}
+        for future in as_completed(futures):
+            index = futures[future]
+            results[index], metadata[index] = future.result()
+
+    assert len(results) == len(inputs), f'results = {len(results)} inputs = {len(inputs)} {results=}'
+    # results = {i: r for r, (_, i) in zip(results, inputs)}
+
+    return results, metadata
+
+
+def codegen_metrics(
+    samples_list,
+    generations_list,
+    k_list=[1, 5, 10, 20, 40, 50, 75, 100, 125, 150, 200, 500, 1000],
+    num_process_evaluate=16,
+    timeout=6,
+    debug=False,
+):
+
+    samples_linear = []
+    generations_linear = []
+    remap_index = []
+    results = defaultdict(list)
+    metadatas = defaultdict(list)
+    for idx, (sample, generation_list) in enumerate(zip(samples_list, generations_list)):
+        assert isinstance(generation_list, list), generations_list[0]
+        for generation in generation_list:
+            assert isinstance(generation, str), generations_list[0]
+            samples_linear.append(sample)
+            generations_linear.append([generation])
+            remap_index.append(idx)
+
+    results_linear, metadatas_linear = evaluate_generations(
+        samples_linear,
+        generations_linear,
+        debug=debug,
+        num_process_evaluate=num_process_evaluate,
+        timeout=timeout,
+    )
+
+    for idx, sub_results in sorted(results_linear.items(), key=lambda x: x[0]):
+        results[remap_index[idx]].append(sub_results[0])
+
+    for idx, sub_metadatas in sorted(metadatas_linear.items(), key=lambda x: x[0]):
+        metadatas[remap_index[idx]].append(sub_metadatas[0])
+
+    metrics = compute_metrics_from_results(results, k_list=k_list)
+
+    final_metadata = []
+    for key in sorted(list(metadatas.keys())):
+        final_metadata.append(metadatas[key])
+    for i in range(len(final_metadata)):
+        if type(final_metadata[i]) is not list:
+            final_metadata[i] = [json.dumps(final_metadata[i])]
+        else:
+            final_metadata[i] = [json.dumps(x) for x in final_metadata[i]]
+
+        assert len(final_metadata[i]) == len(generations_list[0]), f'{len(final_metadata[i])=}'
+
+    return [metrics, results, final_metadata]

evalscope/benchmarks/live_code_bench/execute_utils.py

@@ -0,0 +1,267 @@
+# Copyright 2020 The HuggingFace Datasets Authors and the
+# current dataset script contributor.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This code is adapted from OpenAI's release
+# https://github.com/openai/human-eval/blob/master/human_eval/execution.py
+
+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import tempfile
+
+BASE_IMPORTS = """from itertools import accumulate, chain, combinations, count, permutations, product, groupby, islice, repeat
+from copy import deepcopy
+from string import ascii_lowercase
+from math import floor, log2, log10, sqrt, comb, gcd, ceil, inf, isqrt
+from collections import defaultdict, deque, Counter
+from bisect import bisect, bisect_left, bisect_right, insort
+from heapq import heappush, heappop, heapify, merge
+from functools import reduce, cache, lru_cache
+from random import randrange, shuffle
+from operator import itemgetter, sub
+from re import search as re_search  # Assuming 're' refers to a regex search
+from os.path import commonprefix
+from typing import List, Tuple, Dict, Set, Optional, Union, Any, Callable, Iterable, Iterator, Generator
+import copy
+import string
+import math
+import collections
+import bisect
+import heapq
+import functools
+import random
+import itertools
+import operator
+import re
+import numpy as np
+import pandas as pd
+from math import log, prod  # 'log' and 'prod' are functions in the math module
+from collections import deque, defaultdict, Counter, OrderedDict
+from itertools import accumulate, permutations, combinations, product, groupby, islice, chain, repeat, zip_longest, cycle
+from functools import lru_cache, reduce, partial
+# from sortedcontainers import SortedList, SortedDict, SortedSet
+# import sortedcontainers
+from operator import iand
+import sys
+"""  # noqa: E501
+
+
+def codeexecute_check_correctness(check_program, timeout=3):
+    """Evaluates the functional correctness of a completion by running the test
+    suite provided in the problem.
+
+    :param completion_id: an optional completion ID so we can match
+        the results later even if execution finishes asynchronously.
+    """
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(target=unsafe_execute, args=(check_program, result, timeout))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append('timed out')
+
+    return result[0] == 'passed'
+
+
+def unsafe_execute(check_program, result, timeout):
+
+    with create_tempdir():
+
+        # These system calls are needed when cleaning up tempdir.
+        import os
+        import shutil
+
+        rmtree = shutil.rmtree
+        rmdir = os.rmdir
+        chdir = os.chdir
+
+        # Disable functionalities that can make destructive changes
+        # to the test.
+        reliability_guard()
+
+        # Run program.
+        try:
+            exec_globals = {}
+            with swallow_io():
+                with time_limit(timeout):
+                    exec(check_program, exec_globals)
+            result.append('passed')
+        except TimeoutException:
+            result.append('timed out')
+        except BaseException as e:
+            result.append(f'failed: {e}')
+
+        # Needed for cleaning up.
+        shutil.rmtree = rmtree
+        os.rmdir = rmdir
+        os.chdir = chdir
+
+
+@contextlib.contextmanager
+def time_limit(seconds):
+
+    def signal_handler(signum, frame):
+        raise TimeoutException('Timed out!')
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """StringIO that throws an exception when it's read from."""
+
+    def read(self, *args, **kwargs):
+        raise OSError
+
+    def readline(self, *args, **kwargs):
+        raise OSError
+
+    def readlines(self, *args, **kwargs):
+        raise OSError
+
+    def readable(self, *args, **kwargs):
+        """Returns True if the IO object can be read."""
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = 'stdin'
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == '.':
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes=None):
+    """This disables various destructive functions and prevents the generated
+    code from interfering with the test (e.g. fork bomb, killing other
+    processes, removing filesystem files, etc.)
+
+    WARNING This function is NOT a security sandbox. Untrusted code, including,
+    model- generated code, should not be blindly executed outside of one. See
+    the Codex paper for more information about OpenAI's code sandbox, and
+    proceed with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == 'Darwin':
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ['OMP_NUM_THREADS'] = '1'
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__['help'] = None
+
+    import sys
+
+    sys.modules['ipdb'] = None
+    sys.modules['joblib'] = None
+    sys.modules['resource'] = None
+    sys.modules['psutil'] = None
+    sys.modules['tkinter'] = None

evalscope/benchmarks/live_code_bench/extract_utils.py

@@ -0,0 +1,70 @@
+# Copyright LiveCodeBench @ 2024,
+
+import re
+
+
+def extract_code_generation(model_output: str, model_type: str = 'chat'):
+    # modified from
+    outputlines = model_output.split('\n')
+    # TODO: handle codellama
+
+    if model_type == 'base':
+        return model_output.strip()
+    elif model_type == 'chat':
+        indexlines = [i for i, line in enumerate(outputlines) if '```' in line]
+    else:
+        raise ValueError(f'Invalid mode type: {model_type}')
+
+    if len(indexlines) < 2:
+        return ''
+    return '\n'.join(outputlines[indexlines[0] + 1:indexlines[1]])
+
+
+def extract_code_execution(model_output: str, cot: bool = False):
+    pattern = r'\[PYTHON\](.*?)\[\/PYTHON\]'
+    matches = re.findall(pattern, model_output, re.DOTALL)
+    if matches:
+        # fetch the last one
+        model_output = matches[-1]
+
+    if '[PYTHON]' in model_output:
+        model_output
+    if cot:
+        if '[ANSWER]' in model_output:
+            model_output = model_output.split('[ANSWER]')[1].strip()
+    if '==' in model_output:
+        model_output = model_output.split('==')[1].strip()
+    if '[/ANSWER]' in model_output:
+        model_output = model_output.split('[/ANSWER]')[0].strip()
+    else:
+        model_output = model_output.split('\n')[0].strip()
+    return model_output.strip()
+
+
+def extract_test_output_code(model_output: str):
+    outputlines = model_output.split('\n')
+    # find the last line startwith assert...
+    indexlines = [i for i, line in enumerate(outputlines) if line.startswith('assert')]
+    if indexlines:
+        return outputlines[indexlines[-1]]
+
+    # TODO: handle codellama format
+    # if lmstyle and lmstyle == LMStyle.CodeLLaMaInstruct:
+    #     indexlines = \
+    # [i for i, line in enumerate(outputlines) if "PYTHON]" in line]
+    # else:
+
+    # first try to extract ```python if not then try ```
+    indexlines = [i for i, line in enumerate(outputlines) if '```python' in line or '```Python' in line]
+    if indexlines:
+        start_index = indexlines[0]
+    else:
+        start_index = None
+    indexlines = [i for i, line in enumerate(outputlines) if '```' in line]
+    if start_index is not None:
+        indexlines = [i for i in indexlines if i > start_index]
+        indexlines = [start_index] + indexlines
+
+    if len(indexlines) < 2:
+        return ''
+    return '\n'.join(outputlines[indexlines[0] + 1:indexlines[1]])

evalscope/benchmarks/live_code_bench/live_code_bench_adapter.py

@@ -0,0 +1,90 @@
+from tqdm import tqdm
+
+from evalscope.benchmarks import Benchmark, DataAdapter
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+@Benchmark.register(
+    name='live_code_bench',
+    pretty_name='Live Code Bench',
+    dataset_id='AI-ModelScope/code_generation_lite',
+    subset_list=['release_latest'],
+    metric_list=['Pass@1'],
+    few_shot_num=0,
+    train_split=None,
+    eval_split='test',
+    extra_params={
+        'start_date': None,
+        'end_date': None,
+        'num_process_evaluate': 1,
+        'timeout': 6
+    },
+    system_prompt=
+    'You are an expert Python programmer. You will be given a question (problem specification) and will generate a correct Python program that matches the specification and passes all tests. You will NOT return anything except for the program.',  # noqa: E501
+    prompt_template=
+    '### Question:\n{question_content}\n\n{format_prompt} ### Answer: (use the provided format with backticks)\n\n',  # noqa: E501
+)
+class LiveCodeBenchAdapter(DataAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        extra_params = kwargs.get('extra_params', {})
+
+        self.num_process_evaluate = extra_params.get('num_process_evaluate', 1)
+        self.timeout = extra_params.get('timeout', 6)
+        self.start_date = extra_params.get('start_date')
+        self.end_date = extra_params.get('end_date')
+
+    def load(self, **kwargs) -> dict:
+        from .load_utils import filter_date, transform
+
+        # Note: need trust_remote_code=True to load the python script
+        dataset_dict = super().load(trust_remote_code=True, **kwargs)
+        new_dataset_dict = {}
+        for subset_key, dataset in dataset_dict.items():
+            datasets = dataset[self.eval_split]
+            filtered_datasets = filter_date(datasets, start_date=self.start_date, end_date=self.end_date)
+
+            transformed_datasets = [transform(item) for item in tqdm(filtered_datasets, desc='Transforming data')]
+            new_dataset_dict[subset_key] = {self.eval_split: transformed_datasets}
+        return new_dataset_dict
+
+    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
+        """
+        Generate the prompt for the model input.
+        """
+        format_prompt = input_d['format_prompt']
+        question_content = input_d['question_content']
+        full_prompt = self.prompt_template.format(question_content=question_content, format_prompt=format_prompt)
+
+        return self.gen_prompt_data(full_prompt)
+
+    def get_gold_answer(self, input_d: dict) -> str:
+        # Extract the gold answer from the input dict.
+        return input_d
+
+    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
+        """
+        Parse the model output to get the answer. Could be the best choice index.
+        """
+        return result
+
+    def match(self, gold: dict, pred: str) -> float:
+        from .evaluate_utils import codegen_metrics
+        from .extract_utils import extract_code_generation
+
+        ext_pred = extract_code_generation(pred)
+
+        references = [{'input_output': gold['evaluation_sample']}]
+        predictions = [[ext_pred]]
+        metrics, eval_results, final_metadata = codegen_metrics(
+            references,
+            predictions,
+            k_list=[1],
+            num_process_evaluate=self.num_process_evaluate,
+            timeout=self.timeout,
+        )
+        return metrics['pass@1'] / 100  # convert to point scale

evalscope/benchmarks/live_code_bench/load_utils.py

@@ -0,0 +1,71 @@
+import base64
+import json
+import pickle
+import zlib
+from datetime import datetime
+
+from evalscope.benchmarks.live_code_bench.prompts import CodeGenerationPromptConstants
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+
+def transform(item):
+    # Define the dataitem mapping logic
+
+    # starter_code
+    if item['starter_code']:
+        format_prompt = f'### Format: {CodeGenerationPromptConstants.FORMATTING_MESSAGE_WITH_STARTER_CODE}\n'  # noqa: E501
+        format_prompt += f"```python\n{item['starter_code']}\n```\n\n"
+    else:
+        format_prompt = f'### Format: {CodeGenerationPromptConstants.FORMATTING_WITHOUT_STARTER_CODE}\n'  # noqa: E501
+        format_prompt += '```python\n# YOUR CODE HERE\n```\n\n'
+
+    item['format_prompt'] = format_prompt
+
+    # load test cases
+    public_test_cases = item['public_test_cases']
+    public_test_cases = json.loads(item['public_test_cases'])
+
+    private_test_cases = item['private_test_cases']
+    try:
+        private_test_cases = json.loads(item['private_test_cases'])
+    except Exception as e:  # noqa: F841
+        private_test_cases = json.loads(
+            pickle.loads(zlib.decompress(base64.b64decode(private_test_cases.encode('utf-8'))  # type: ignore
+                                         )))  # type: ignore
+
+    # load metadata
+    metadata = json.loads(item['metadata'])
+    evaluation_sample = json.dumps({
+        'inputs': [t['input'] for t in public_test_cases + private_test_cases],
+        'outputs': [t['output'] for t in public_test_cases + private_test_cases],
+        'fn_name': metadata.get('func_name', None),
+    })
+    item['evaluation_sample'] = evaluation_sample
+
+    return item
+
+
+def filter_date(dataset, start_date=None, end_date=None):
+    new_dataset = []
+
+    for item in dataset:
+        contest_date = datetime.fromisoformat(item['contest_date'])
+        if start_date is not None:
+            p_start_date = datetime.strptime(start_date, '%Y-%m-%d')
+            if p_start_date > contest_date:
+                continue
+
+        if end_date is not None:
+            p_end_date = datetime.strptime(end_date, '%Y-%m-%d')
+            if p_end_date < contest_date:
+                continue
+
+        new_dataset.append(item)
+
+    if start_date or end_date:
+        logger.info(
+            f'Filtered dataset with start_date: {start_date}, end_date: {end_date}, remaining items: {len(new_dataset)}'
+        )
+    return new_dataset