evalscope 0.10.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/hle/hle_adapter.py

@@ -0,0 +1,153 @@
+import re
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages import ChatMessage, ChatMessageSystem, ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.logger import get_logger
+
+# flake8: noqa
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'Biology/Medicine',
+    'Chemistry',
+    'Computer Science/AI',
+    'Engineering',
+    'Humanities/Social Science',
+    'Math',
+    'Physics',
+    'Other',
+]
+
+ANSWER_TYPE_EXACT_MATCH = 'exactMatch'
+ANSWER_TYPE_MULTIPLE_CHOICE = 'multipleChoice'
+
+# System prompt constants
+SYSTEM_EXACT_ANSWER = 'Your response should be in the following format:\nExplanation: {your explanation for your final answer}\nExact Answer: {your succinct, final answer}\nConfidence: {your confidence score between 0% and 100% for your answer}'
+
+SYSTEM_MC = 'Your response should be in the following format:\nExplanation: {your explanation for your answer choice}\nAnswer: {your chosen answer}\nConfidence: {your confidence score between 0% and 100% for your answer}'
+
+JUDGE_PROMPT = """Judge whether the following [response] to [question] is correct or not based on the precise and unambiguous [correct_answer] below.
+
+[question]: {question}
+
+[response]: {response}
+
+[correct_answer]: {correct_answer}
+
+Your judgment must focus only on if there are meaningful differences between [correct_answer] and the [response]. Do not comment on any background to the problem, do not attempt to solve the problem, do not argue for any answer different than [correct_answer], focus only on whether the answers match. Explain why the [response] is correct or incorrect based on [correct_answer] in one or two sentences. Finally, write your answer in the format 'GRADE: C' for correct answer or 'GRADE: I' for incorrect answer.
+"""
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='hle',
+        pretty_name="Humanity's-Last-Exam",
+        tags=[Tags.KNOWLEDGE, Tags.QA],
+        description='Humanity\'s Last Exam (HLE) is a language model benchmark consisting of 2,500 '
+        'questions across a broad range of subjects. It was created jointly by the Center '
+        'for AI Safety and Scale AI. The benchmark classifies the questions into the '
+        'following broad subjects: mathematics (41%), physics (9%), biology/medicine (11%), '
+        'humanities/social science (9%), computer science/artificial intelligence (10%), '
+        'engineering (4%), chemistry (7%), and other (9%). Around 14% of the questions '
+        'require the ability to understand both text and images, i.e., multi-modality. '
+        '24% of the questions are multiple-choice; the rest are short-answer, exact-match questions. \n'
+        '**To evaluate the performance of model without multi-modality capabilities, '
+        'please set the `extra_params["include_multi_modal"]` to `False`.**',  # noqa: E501
+        dataset_id='cais/hle',
+        subset_list=SUBSET_LIST,
+        metric_list=['acc'],
+        eval_split='test',
+        prompt_template='{question}',
+        extra_params={'include_multi_modal': True}
+    )
+)
+class HLEAdapter(DefaultDataAdapter):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+
+        self._use_llm_judge = True  # Use LLM as a judge by default
+        self.reformat_subset = True
+        self.include_multi_modal = self.extra_params.get('include_multi_modal', True)
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        answer_type = record['answer_type']
+        system_prompt = (SYSTEM_EXACT_ANSWER if answer_type == ANSWER_TYPE_EXACT_MATCH else SYSTEM_MC)
+        text_content = ContentText(text=record['question'])
+
+        content: List[Content] = [text_content]
+        if record['image']:
+            image_content = ContentImage(image=record['image'])
+            content.append(image_content)
+
+        messages: List[ChatMessage] = [
+            ChatMessageSystem(content=system_prompt),
+            ChatMessageUser(content=content),
+        ]
+        return Sample(
+            input=messages,
+            subset_key=record['category'],
+            metadata={
+                'uid': record['id'],
+                'author_name': record['author_name'],
+                'rationale': record['rationale'],
+                'raw_subject': record['raw_subject'],
+                'category': record['category'],
+                'has_image': bool(record['image']),
+            },
+            target=record['answer'],
+        )
+
+    def sample_filter(self, sample):
+        if not self.include_multi_modal:
+            if sample.metadata is not None and sample.metadata['has_image']:
+                return False
+        return True
+
+    def llm_match_score(
+        self,
+        original_prediction: str,
+        filtered_prediction: str,
+        reference: str,
+        task_state: TaskState,
+    ) -> Score:
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+
+        confidence = 100
+        if task_state.output and task_state.output.completion:
+            confidence_match = re.search(r'confidence:\s*(\d+)', task_state.output.completion, re.IGNORECASE)
+            if confidence_match:
+                confidence = int(confidence_match.group(1))
+
+        judge_prompt = JUDGE_PROMPT.format(
+            question=task_state.input_text, response=filtered_prediction, correct_answer=reference
+        )
+
+        # Request judge and obtain score
+        judge_response = self.llm_judge.judge(prompt=judge_prompt)
+
+        # Parse judge response to get accuracy score
+        accuracy_score = re.search(r'GRADE:\s*([CI])', judge_response, re.IGNORECASE)
+        if accuracy_score:
+            score.value = {
+                'acc': 1.0 if accuracy_score.group(1) == 'C' else 0.0,
+            }
+        score.explanation = f'LLM judge: {judge_response}'
+        score.metadata = {
+            'source': 'llm_judge',
+            'judge_strategy': self.judge_strategy,
+            'model': self.llm_judge.model_id,
+            'confidence': confidence,
+        }
+        score.main_score_name = 'acc'
+        return score

evalscope/benchmarks/humaneval/humaneval_adapter.py

@@ -1,104 +1,96 @@
 # Copyright (c) Alibaba, Inc. and its affiliates.
+# flake8: noqa: E501
 import re
-
-from evalscope.benchmarks import Benchmark, DataAdapter
-from evalscope.metrics import Pass1
-from evalscope.models import ChatGenerationModelAdapter
+from typing import Any, Dict
+
+from evalscope.api.benchmark import BenchmarkMeta, DefaultDataAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator import TaskState
+from evalscope.api.messages.chat_message import ChatMessageUser
+from evalscope.api.metric import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
 from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
-# Example:
-# {"task_id": "HumanEval/0", "prompt": "from typing import List\n\n\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\n    \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\n    given threshold.\n    >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\n    False\n    >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\n    True\n    \"\"\"\n", "entry_point": "has_close_elements", "canonical_solution": "    for idx, elem in enumerate(numbers):\n        for idx2, elem2 in enumerate(numbers):\n            if idx != idx2:\n                distance = abs(elem - elem2)\n                if distance < threshold:\n                    return True\n\n    return False\n", "test": "\n\nMETADATA = {\n    'author': 'jt',\n    'dataset': 'test'\n}\n\n\ndef check(candidate):\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\n    assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\n\n"}  # noqa
-
 
-@Benchmark.register(
-    name='humaneval',
-    dataset_id='modelscope/humaneval',
-    model_adapter=ChatGenerationModelAdapter,
-    subset_list=['openai_humaneval'],
-    metric_list=[Pass1],
-    few_shot_num=0,
-    train_split=None,
-    eval_split='test',
-    prompt_template='',
+@register_benchmark(
+    BenchmarkMeta(
+        name='humaneval',
+        pretty_name='HumanEval',
+        tags=[Tags.CODING],
+        description=
+        'HumanEval is a benchmark for evaluating the ability of code generation models to write Python functions based on given specifications. It consists of programming tasks with a defined input-output behavior. '
+        '**By default the code is executed in local environment. We recommend using sandbox execution to safely run and evaluate the generated code, please refer to the [documentation](https://evalscope.readthedocs.io/en/latest/user_guides/sandbox.html) for more details.**',  # noqa: E501
+        dataset_id='opencompass/humaneval',
+        subset_list=['openai_humaneval'],
+        aggregation='mean_and_pass_at_k',
+        eval_split='test',
+        prompt_template=
+        'Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{question}',
+        review_timeout=4,
+    )
 )
-class HumanevalAdapter(DataAdapter):
+class HumanevalAdapter(DefaultDataAdapter):
     """
-    A placeholder for humaneval adapter, see HumanevalEvaluator for implementation.
+    HumanEval adapter using the new data processing framework.
     """
 
-    def __init__(self, **kwargs):
-        try:
-            from human_eval.data import stream_jsonl, write_jsonl
-            from human_eval.evaluation import check_correctness
-        except ImportError:
-            raise ImportError('Please install human_eval:'
-                              'https://github.com/openai/human-eval/tree/master#installation , '
-                              'Note that you need to enable the execution code in the human_eval/execution.py first.')
-
-        self.k = [1]
-        self.num_workers = 4
-        self.timeout = 4.0
-
-        self.read_problems_func = stream_jsonl
-        self.write_jsonl_func = write_jsonl
-        self.eval_func = check_correctness
-
-        super().__init__(**kwargs)
-
-    def load_from_disk(self, dataset_name_or_path, subset_list, work_dir, **kwargs) -> dict:
-        data_dict = {}
-        for subset_name in subset_list:
-            data_dict[subset_name] = {}
-            # [{'task_id': '', 'prompt': '', 'entry_point': '', 'canonical_solution': '', 'test': ''}, ...]
-            data_dict[subset_name][self.eval_split] = [task for task in self.read_problems_func(dataset_name_or_path)]
-
-        return data_dict
-
-    def gen_prompt(self, input_d: dict, few_shot_list: list, **kwargs) -> dict:
-        """
-        Generate prompt for the model.
-
-        Args:
-            input_d (dict): The raw input. A single data format of the Humaneval:
-            {'task_id': '', 'prompt': '', 'entry_point': '', 'canonical_solution': '', 'test': ''}
-        """
-        full_prompt = input_d['prompt']
-        full_prompt = f'Complete the following python code:\n{full_prompt}' if self.prompt_template else full_prompt
-
-        return {'data': [full_prompt], 'system_prompt': self.prompt_template}
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        """Convert a data record to a Sample object."""
+        query = record['prompt']
+        full_prompt = self.prompt_template.format(question=query)
+
+        return Sample(
+            input=[ChatMessageUser(content=full_prompt)],
+            target=record['canonical_solution'],
+            metadata={
+                'task_id': record['task_id'],
+                'entry_point': record['entry_point'],
+                'prompt': record['prompt'],
+                'test': record['test'],
+            }
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        """Extract code from the prediction."""
+        return self._postprocess(prediction)
 
     @classmethod
     def _postprocess(cls, text: str) -> str:
-        if '```' in text:
-            blocks = re.findall(r'```(.*?)```', text, re.DOTALL)
-            if len(blocks) == 0:
-                text = text.split('```')[1]  # fall back to default strategy
-            else:
-                text = blocks[0]  # fetch the first code block
-                if not text.startswith('\n'):  # in case starting with ```python
-                    text = text[max(text.find('\n') + 1, 0):]
-        if text.strip().startswith('from') or text.strip().startswith('import'):
-            def_idx = text.find('def')
-            if def_idx != -1:
-                text = text[max(text.find('\n', def_idx) + 1, 0):]
-        text = text.split('\n\n')[0]
-        if text.strip().startswith('def'):
-            text = '\n'.join(text.split('\n')[1:])
-        if not text.startswith('    '):
-            if text.startswith(' '):
-                text = '    ' + text.lstrip()
-            else:
-                text = '\n'.join(['    ' + line for line in text.split('\n')])
+        """Extract code from markdown code blocks."""
+        blocks = re.findall(r'```\w*\n(.*?)```', text, re.DOTALL)
+        if len(blocks) >= 1:
+            text = blocks[0]
         return text
 
-    def parse_pred_result(self, result: str, raw_input_d: dict = None, eval_type: str = 'checkpoint') -> str:
-        return self._postprocess(result)
-
-    def get_gold_answer(self, input_d: dict) -> str:
-        return input_d
-
-    def match(self, gold: str, pred: str) -> float:
-        res = self.eval_func(gold, pred, self.timeout)
-        return float(res['passed'])
+    def match_score(
+        self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
+    ) -> Score:
+
+        score = Score(
+            extracted_prediction=filtered_prediction,
+            prediction=original_prediction,
+        )
+        problem = task_state.metadata
+        completion = filtered_prediction
+
+        if not self.use_sandbox:
+            from .utils import check_correctness
+
+            # Execute the code and check correctness
+            res = check_correctness(problem=problem, completion=completion, timeout=self.review_timeout)
+            passed = res['passed']
+        else:
+            check_program = (
+                problem['prompt'] + completion + '\n' + problem['test'] + '\n' + f"check({problem['entry_point']})"
+            )
+            res = self.execute_code_in_sandbox(code=check_program, timeout=self.review_timeout, language='python')
+            passed = res.get('status') == 'success'
+        # Set score values
+        score.value = {'acc': passed}
+        score.metadata = {'task_id': problem['task_id'], 'timeout': self.review_timeout, 'execution_result': res}
+        score.main_score_name = 'acc'
+
+        return score

evalscope/benchmarks/humaneval/utils.py

@@ -0,0 +1,235 @@
+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import tempfile
+from typing import Dict, Optional
+
+
+def unsafe_execute(problem: Dict, completion: str, timeout: float, result):
+    with create_tempdir():
+
+        # These system calls are needed when cleaning up tempdir.
+        import os
+        import shutil
+
+        rmtree = shutil.rmtree
+        rmdir = os.rmdir
+        chdir = os.chdir
+
+        # Disable functionalities that can make destructive changes to the test.
+        reliability_guard()
+
+        # Construct the check program and run it.
+        check_program = (
+            problem['prompt'] + completion + '\n' + problem['test'] + '\n' + f"check({problem['entry_point']})"
+        )
+
+        try:
+            exec_globals = {}
+            with swallow_io():
+                with time_limit(timeout):
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network. For more
+                    # information on how OpenAI sandboxes its code, see the accompanying paper.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                    exec(check_program, exec_globals)
+            result.append('passed')
+        except TimeoutException:
+            result.append('timed out')
+        except BaseException as e:
+            result.append(f'failed: {e}')
+
+        # Needed for cleaning up.
+        shutil.rmtree = rmtree
+        os.rmdir = rmdir
+        os.chdir = chdir
+
+
+def check_correctness(problem: Dict, completion: str, timeout: float, completion_id: Optional[int] = None) -> Dict:
+    """
+    Evaluates the functional correctness of a completion by running the test
+    suite provided in the problem.
+
+    :param completion_id: an optional completion ID so we can match
+        the results later even if execution finishes asynchronously.
+    """
+
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(target=unsafe_execute, args=(problem, completion, timeout, result))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append('timed out')
+
+    return dict(
+        task_id=problem['task_id'],
+        passed=result[0] == 'passed',
+        result=result[0],
+        completion_id=completion_id,
+    )
+
+
+@contextlib.contextmanager
+def time_limit(seconds: float):
+
+    def signal_handler(signum, frame):
+        raise TimeoutException('Timed out!')
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """StringIO that throws an exception when it's read from"""
+
+    def read(self, *args, **kwargs):
+        raise IOError
+
+    def readline(self, *args, **kwargs):
+        raise IOError
+
+    def readlines(self, *args, **kwargs):
+        raise IOError
+
+    def readable(self, *args, **kwargs):
+        """Returns True if the IO object can be read."""
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = 'stdin'
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == '.':
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes: Optional[int] = None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == 'Darwin':
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ['OMP_NUM_THREADS'] = '1'
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__['help'] = None
+
+    import sys
+
+    sys.modules['ipdb'] = None
+    sys.modules['joblib'] = None
+    sys.modules['resource'] = None
+    sys.modules['psutil'] = None
+    sys.modules['tkinter'] = None