evalscope 1.0.0__py3-none-any.whl → 1.0.2__py3-none-any.whl

evalscope/benchmarks/healthbench/utils.py

@@ -0,0 +1,102 @@
+import json
+import re
+from collections import defaultdict
+
+from evalscope.utils import get_logger
+
+logger = get_logger()
+
+
+def parse_json_to_dict(json_string: str) -> dict:
+    # Remove markdown-style ```json``` markers if present
+    json_cleaned = re.sub(r'^```json\s*|\s*```$', '', json_string.strip())
+
+    try:
+        return json.loads(json_cleaned)
+    except json.JSONDecodeError as e:
+        logger.warning(f'JSON decoding failed: {e}')
+        return {}
+
+
+class RubricItem:
+
+    def __init__(self, criterion: str, points: float, tags: list[str]):
+        self.criterion = criterion
+        self.points = points
+        self.tags = tags
+
+    def __str__(self):
+        return f'[{self.points}] {self.criterion}'
+
+    def to_dict(self):
+        return {
+            'criterion': self.criterion,
+            'points': self.points,
+            'tags': self.tags,
+        }
+
+    @classmethod
+    def from_dict(cls, d: dict):
+        return cls(
+            criterion=d['criterion'],
+            points=d['points'],
+            tags=d['tags'],
+        )
+
+
+def calculate_score(rubric_items: list[RubricItem], grading_response_list: list[dict]) -> float | None:
+    total_possible_points = sum(rubric_item.points for rubric_item in rubric_items if rubric_item.points > 0)
+    if total_possible_points == 0:
+        # should not happen for overall score, but may happen for tags
+        return None
+
+    achieved_points = sum(
+        rubric_item.points
+        for rubric_item, grading_response in zip(rubric_items, grading_response_list, strict=True)
+        if grading_response['criteria_met']
+    )
+    overall_score = achieved_points / total_possible_points
+    return overall_score
+
+
+def calculate_rubric_tag_scores(rubric_items: list[RubricItem], grading_response_list: list[dict]) -> dict[str, float]:
+    rubric_tag_items_grades = defaultdict(list)
+    axis_grades = defaultdict(list)
+    for rubric_item, grading_response in zip(rubric_items, grading_response_list):
+        curr_item_tags = set()  # Ensure no duplicates in a rubric item.
+        for tag in rubric_item.tags:
+            rubric_tag_items_grades[tag].append((rubric_item, grading_response))
+            assert tag not in curr_item_tags
+            curr_item_tags.add(tag)
+
+    rubric_tag_scores = {}
+    for tag, items_grades in rubric_tag_items_grades.items():
+        items, grades = zip(*items_grades)
+        score = calculate_score(items, grades)
+        if score is not None:  # implies at least one positive criterion
+            rubric_tag_scores[tag] = score
+            if tag.startswith('axis:'):
+                axis_grades[tag.split(':')[1]] = score
+
+    return rubric_tag_scores, axis_grades
+
+
+def construct_readable_explanation(rubric_items: list[RubricItem], grading_response_list: list[dict]) -> str:
+    rubric_items_with_grades = []
+    readable_explanation_list = []
+    for rubric_item, grading_response in zip(rubric_items, grading_response_list):
+        explanation = grading_response.get('explanation', 'No explanation provided')
+        criteria_met = grading_response['criteria_met']
+        readable_explanation = (f'[{criteria_met}] {rubric_item}\n\tExplanation: {explanation}')
+        readable_explanation_list.append(readable_explanation)
+        rubric_items_with_grades.append({
+            **rubric_item.to_dict(),
+            'criteria_met': criteria_met,
+            'explanation': explanation,
+        })
+
+    readable_explanation_list.sort(key=lambda x: x.startswith('[False]'), reverse=True)
+    readable_explanation_str = '\n\n'.join(readable_explanation_list)
+    readable_explanation_str = f'\n\n{readable_explanation_str}'
+
+    return readable_explanation_str

evalscope/benchmarks/humaneval/humaneval_adapter.py

@@ -14,9 +14,6 @@ from evalscope.utils.logger import get_logger
 
 logger = get_logger()
 
-# Example:
-# {"task_id": "HumanEval/0", "prompt": "from typing import List\n\n\ndef has_close_elements(numbers: List[float], threshold: float) -> bool:\n    \"\"\" Check if in given list of numbers, are any two numbers closer to each other than\n    given threshold.\n    >>> has_close_elements([1.0, 2.0, 3.0], 0.5)\n    False\n    >>> has_close_elements([1.0, 2.8, 3.0, 4.0, 5.0, 2.0], 0.3)\n    True\n    \"\"\"\n", "entry_point": "has_close_elements", "canonical_solution": "    for idx, elem in enumerate(numbers):\n        for idx2, elem2 in enumerate(numbers):\n            if idx != idx2:\n                distance = abs(elem - elem2)\n                if distance < threshold:\n                    return True\n\n    return False\n", "test": "\n\nMETADATA = {\n    'author': 'jt',\n    'dataset': 'test'\n}\n\n\ndef check(candidate):\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.3) == True\n    assert candidate([1.0, 2.0, 3.9, 4.0, 5.0, 2.2], 0.05) == False\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.95) == True\n    assert candidate([1.0, 2.0, 5.9, 4.0, 5.0], 0.8) == False\n    assert candidate([1.0, 2.0, 3.0, 4.0, 5.0, 2.0], 0.1) == True\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 1.0) == True\n    assert candidate([1.1, 2.2, 3.1, 4.1, 5.1], 0.5) == False\n\n"}  # noqa
-
 
 @register_benchmark(
     BenchmarkMeta(
@@ -31,10 +28,7 @@ logger = get_logger()
         eval_split='test',
         prompt_template=
         'Read the following function signature and docstring, and fully implement the function described. Your response should only contain the code for this function.\n{question}',
-        extra_params={
-            'num_workers': 4,
-            'timeout': 4
-        },
+        review_timeout=4,
     )
 )
 class HumanevalAdapter(DefaultDataAdapter):
@@ -42,27 +36,6 @@ class HumanevalAdapter(DefaultDataAdapter):
     HumanEval adapter using the new data processing framework.
     """
 
-    def __init__(self, **kwargs):
-        try:
-            from human_eval.data import stream_jsonl, write_jsonl
-            from human_eval.evaluation import check_correctness
-        except ImportError:
-            raise ImportError(
-                'Please install human_eval:'
-                'https://github.com/openai/human-eval/tree/master#installation , '
-                'Note that you need to enable the execution code in the human_eval/execution.py first.'
-            )
-        super().__init__(**kwargs)
-
-        extra_params = kwargs.get('extra_params', {})
-        self.k = [1]
-        self.num_workers = extra_params.get('num_workers', 4)
-        self.timeout = extra_params.get('timeout', 4)
-
-        self.read_problems_func = stream_jsonl
-        self.write_jsonl_func = write_jsonl
-        self.eval_func = check_correctness
-
     def record_to_sample(self, record: Dict[str, Any]) -> Sample:
         """Convert a data record to a Sample object."""
         query = record['prompt']
@@ -94,18 +67,29 @@ class HumanevalAdapter(DefaultDataAdapter):
     def match_score(
         self, original_prediction: str, filtered_prediction: str, reference: str, task_state: TaskState
     ) -> Score:
+
         score = Score(
             extracted_prediction=filtered_prediction,
             prediction=original_prediction,
         )
-
-        # Execute the code and check correctness
-        res = self.eval_func(task_state.metadata, filtered_prediction, self.timeout)
-        passed = res['passed']
-
+        problem = task_state.metadata
+        completion = filtered_prediction
+
+        if not self.use_sandbox:
+            from .utils import check_correctness
+
+            # Execute the code and check correctness
+            res = check_correctness(problem=problem, completion=completion, timeout=self.review_timeout)
+            passed = res['passed']
+        else:
+            check_program = (
+                problem['prompt'] + completion + '\n' + problem['test'] + '\n' + f"check({problem['entry_point']})"
+            )
+            res = self.execute_code_in_sandbox(code=check_program, timeout=self.review_timeout, language='python')
+            passed = res.get('status') == 'success'
+        # Set score values
         score.value = {'pass': passed}
-        score.explanation = res.get('result', 'Code execution completed')
-        score.metadata = {'task_id': task_state.metadata['task_id'], 'timeout': self.timeout, 'execution_result': res}
+        score.metadata = {'task_id': problem['task_id'], 'timeout': self.review_timeout, 'execution_result': res}
         score.main_score_name = 'pass'
 
         return score

evalscope/benchmarks/humaneval/utils.py

@@ -0,0 +1,235 @@
+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import tempfile
+from typing import Dict, Optional
+
+
+def unsafe_execute(problem: Dict, completion: str, timeout: float, result):
+    with create_tempdir():
+
+        # These system calls are needed when cleaning up tempdir.
+        import os
+        import shutil
+
+        rmtree = shutil.rmtree
+        rmdir = os.rmdir
+        chdir = os.chdir
+
+        # Disable functionalities that can make destructive changes to the test.
+        reliability_guard()
+
+        # Construct the check program and run it.
+        check_program = (
+            problem['prompt'] + completion + '\n' + problem['test'] + '\n' + f"check({problem['entry_point']})"
+        )
+
+        try:
+            exec_globals = {}
+            with swallow_io():
+                with time_limit(timeout):
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network. For more
+                    # information on how OpenAI sandboxes its code, see the accompanying paper.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                    exec(check_program, exec_globals)
+            result.append('passed')
+        except TimeoutException:
+            result.append('timed out')
+        except BaseException as e:
+            result.append(f'failed: {e}')
+
+        # Needed for cleaning up.
+        shutil.rmtree = rmtree
+        os.rmdir = rmdir
+        os.chdir = chdir
+
+
+def check_correctness(problem: Dict, completion: str, timeout: float, completion_id: Optional[int] = None) -> Dict:
+    """
+    Evaluates the functional correctness of a completion by running the test
+    suite provided in the problem.
+
+    :param completion_id: an optional completion ID so we can match
+        the results later even if execution finishes asynchronously.
+    """
+
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(target=unsafe_execute, args=(problem, completion, timeout, result))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append('timed out')
+
+    return dict(
+        task_id=problem['task_id'],
+        passed=result[0] == 'passed',
+        result=result[0],
+        completion_id=completion_id,
+    )
+
+
+@contextlib.contextmanager
+def time_limit(seconds: float):
+
+    def signal_handler(signum, frame):
+        raise TimeoutException('Timed out!')
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """StringIO that throws an exception when it's read from"""
+
+    def read(self, *args, **kwargs):
+        raise IOError
+
+    def readline(self, *args, **kwargs):
+        raise IOError
+
+    def readlines(self, *args, **kwargs):
+        raise IOError
+
+    def readable(self, *args, **kwargs):
+        """Returns True if the IO object can be read."""
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = 'stdin'
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == '.':
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes: Optional[int] = None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == 'Darwin':
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ['OMP_NUM_THREADS'] = '1'
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__['help'] = None
+
+    import sys
+
+    sys.modules['ipdb'] = None
+    sys.modules['joblib'] = None
+    sys.modules['resource'] = None
+    sys.modules['psutil'] = None
+    sys.modules['tkinter'] = None

evalscope/benchmarks/ifeval/instructions_util.py

@@ -14,7 +14,6 @@
 """Utility library of instructions."""
 
 import functools
-import immutabledict
 import nltk
 import os
 import random
@@ -1551,7 +1550,7 @@ WORD_LIST = [
 ]  # pylint: disable=line-too-long
 
 # ISO 639-1 codes to language names.
-LANGUAGE_CODES = immutabledict.immutabledict({
+LANGUAGE_CODES = {
     'en': 'English',
     'es': 'Spanish',
     'pt': 'Portuguese',
@@ -1582,7 +1581,7 @@ LANGUAGE_CODES = immutabledict.immutabledict({
     'pa': 'Punjabi',
     'ml': 'Malayalam',
     'fi': 'Finnish',
-})
+}
 
 _ALPHABETS = '([A-Za-z])'
 _PREFIXES = '(Mr|St|Mrs|Ms|Dr)[.]'

evalscope/benchmarks/image_edit/gedit/gedit_adapter.py

@@ -0,0 +1,138 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import copy
+import os
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, ImageEditAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessage, ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric.scorer import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import FileConstants, Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'background_change', 'color_alter', 'material_alter', 'motion_change', 'ps_human', 'style_change', 'subject-add',
+    'subject-remove', 'subject-replace', 'text_change', 'tone_transfer'
+]
+
+LANGUAGE_LIST = ['en', 'cn']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='gedit',
+        pretty_name='GEdit-Bench',
+        dataset_id='stepfun-ai/GEdit-Bench',
+        description='GEdit-Bench Image Editing Benchmark, grounded in real-world '
+        'usages is developed to support more authentic and '
+        'comprehensive evaluation of image editing models.',
+        tags=[Tags.IMAGE_EDITING],
+        subset_list=SUBSET_LIST,
+        metric_list=['Semantic Consistency', 'Perceptual Similarity'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',
+        extra_params={'language': f'# language of the instruction, choose from {LANGUAGE_LIST}, default to `en`'}
+    )
+)
+class GEditAdapter(ImageEditAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.language = self.extra_params.get('language', 'en')
+        if self.language not in LANGUAGE_LIST:
+            logger.warning(f"Invalid language '{self.language}', fallback to 'en'")
+            self.language = 'en'
+        self.reformat_subset = True
+        self._use_llm_judge = True
+
+        self.load_prompt()
+
+    def load_prompt(self):
+        from . import vie_prompts
+
+        self.context = vie_prompts._context_no_delimit
+        self.SC_prompt = '\n'.join([
+            self.context, vie_prompts._prompts_0shot_two_image_edit_rule, vie_prompts._prompts_0shot_tie_rule_SC
+        ])
+        self.PQ_prompt = '\n'.join([self.context, vie_prompts._prompts_0shot_rule_PQ])
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        record = copy.deepcopy(record)
+
+        # Process instruction and image
+        instruction = record['instruction']
+        image_bytes = record['input_image']['bytes']
+        input_image = bytes_to_base64(image_bytes, format='png', add_header=True)
+        record['input_image'] = input_image
+        record[FileConstants.ID] = record['key']
+        del record['input_image_raw']
+
+        text_content = ContentText(text=instruction)
+        image_content = ContentImage(image=input_image)
+
+        messages: List[ChatMessage] = [
+            ChatMessageUser(content=[text_content, image_content]),
+        ]
+
+        return Sample(input=messages, subset_key=record['task_type'], metadata=record)
+
+    def sample_filter(self, sample: Sample) -> bool:
+        language = sample.metadata.get('instruction_language', 'en')
+        return super().sample_filter(sample) and language == self.language
+
+    def llm_match_score(self, original_prediction, filtered_prediction, reference, task_state: TaskState) -> Score:
+        import math
+
+        from .utils import mllm_output_to_dict
+
+        metadata = task_state.metadata
+        text_prompt = metadata['instruction']
+        input_image = metadata['input_image']  # base64 image
+        edited_image = metadata[FileConstants.IMAGE_PATH]  # local image path
+        _SC_prompt = self.SC_prompt.replace('<instruction>', text_prompt)
+
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=edited_image,
+            prediction=edited_image,
+        )
+
+        # Build prompts
+        SC_prompt_final = [
+            ChatMessageUser(
+                content=[
+                    ContentImage(image=input_image),
+                    ContentImage(image=edited_image),
+                    ContentText(text=_SC_prompt)
+                ]
+            )
+        ]
+        PQ_prompt_final = [
+            ChatMessageUser(content=[ContentImage(image=edited_image),
+                                     ContentText(text=self.PQ_prompt)])
+        ]
+
+        guess_if_cannot_parse = True
+        result_SC = self.llm_judge.judge(messages=SC_prompt_final)
+        result_PQ = self.llm_judge.judge(messages=PQ_prompt_final)
+        SC_dict = mllm_output_to_dict(result_SC, give_up_parsing=guess_if_cannot_parse)
+        PQ_dict = mllm_output_to_dict(result_PQ, give_up_parsing=guess_if_cannot_parse)
+
+        SC_score = min(SC_dict['score'])
+        PQ_score = min(PQ_dict['score'])
+        O_score = math.sqrt(SC_score * PQ_score)
+
+        score.value = {'Semantic Consistency': SC_score, 'Perceptual Quality': PQ_score, 'Overall': O_score}
+        score.main_score_name = 'Overall'
+        score.metadata = {
+            'SC_dict': SC_dict,
+            'PQ_dict': PQ_dict,
+        }
+        return score