evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/humaneval/utils.py

@@ -0,0 +1,235 @@
+import contextlib
+import faulthandler
+import io
+import multiprocessing
+import os
+import platform
+import signal
+import tempfile
+from typing import Dict, Optional
+
+
+def unsafe_execute(problem: Dict, completion: str, timeout: float, result):
+    with create_tempdir():
+
+        # These system calls are needed when cleaning up tempdir.
+        import os
+        import shutil
+
+        rmtree = shutil.rmtree
+        rmdir = os.rmdir
+        chdir = os.chdir
+
+        # Disable functionalities that can make destructive changes to the test.
+        reliability_guard()
+
+        # Construct the check program and run it.
+        check_program = (
+            problem['prompt'] + completion + '\n' + problem['test'] + '\n' + f"check({problem['entry_point']})"
+        )
+
+        try:
+            exec_globals = {}
+            with swallow_io():
+                with time_limit(timeout):
+                    # WARNING
+                    # This program exists to execute untrusted model-generated code. Although
+                    # it is highly unlikely that model-generated code will do something overtly
+                    # malicious in response to this test suite, model-generated code may act
+                    # destructively due to a lack of model capability or alignment.
+                    # Users are strongly encouraged to sandbox this evaluation suite so that it
+                    # does not perform destructive actions on their host or network. For more
+                    # information on how OpenAI sandboxes its code, see the accompanying paper.
+                    # Once you have read this disclaimer and taken appropriate precautions,
+                    # uncomment the following line and proceed at your own risk:
+                    exec(check_program, exec_globals)
+            result.append('passed')
+        except TimeoutException:
+            result.append('timed out')
+        except BaseException as e:
+            result.append(f'failed: {e}')
+
+        # Needed for cleaning up.
+        shutil.rmtree = rmtree
+        os.rmdir = rmdir
+        os.chdir = chdir
+
+
+def check_correctness(problem: Dict, completion: str, timeout: float, completion_id: Optional[int] = None) -> Dict:
+    """
+    Evaluates the functional correctness of a completion by running the test
+    suite provided in the problem.
+
+    :param completion_id: an optional completion ID so we can match
+        the results later even if execution finishes asynchronously.
+    """
+
+    manager = multiprocessing.Manager()
+    result = manager.list()
+
+    p = multiprocessing.Process(target=unsafe_execute, args=(problem, completion, timeout, result))
+    p.start()
+    p.join(timeout=timeout + 1)
+    if p.is_alive():
+        p.kill()
+
+    if not result:
+        result.append('timed out')
+
+    return dict(
+        task_id=problem['task_id'],
+        passed=result[0] == 'passed',
+        result=result[0],
+        completion_id=completion_id,
+    )
+
+
+@contextlib.contextmanager
+def time_limit(seconds: float):
+
+    def signal_handler(signum, frame):
+        raise TimeoutException('Timed out!')
+
+    signal.setitimer(signal.ITIMER_REAL, seconds)
+    signal.signal(signal.SIGALRM, signal_handler)
+    try:
+        yield
+    finally:
+        signal.setitimer(signal.ITIMER_REAL, 0)
+
+
+@contextlib.contextmanager
+def swallow_io():
+    stream = WriteOnlyStringIO()
+    with contextlib.redirect_stdout(stream):
+        with contextlib.redirect_stderr(stream):
+            with redirect_stdin(stream):
+                yield
+
+
+@contextlib.contextmanager
+def create_tempdir():
+    with tempfile.TemporaryDirectory() as dirname:
+        with chdir(dirname):
+            yield dirname
+
+
+class TimeoutException(Exception):
+    pass
+
+
+class WriteOnlyStringIO(io.StringIO):
+    """StringIO that throws an exception when it's read from"""
+
+    def read(self, *args, **kwargs):
+        raise IOError
+
+    def readline(self, *args, **kwargs):
+        raise IOError
+
+    def readlines(self, *args, **kwargs):
+        raise IOError
+
+    def readable(self, *args, **kwargs):
+        """Returns True if the IO object can be read."""
+        return False
+
+
+class redirect_stdin(contextlib._RedirectStream):  # type: ignore
+    _stream = 'stdin'
+
+
+@contextlib.contextmanager
+def chdir(root):
+    if root == '.':
+        yield
+        return
+    cwd = os.getcwd()
+    os.chdir(root)
+    try:
+        yield
+    except BaseException as exc:
+        raise exc
+    finally:
+        os.chdir(cwd)
+
+
+def reliability_guard(maximum_memory_bytes: Optional[int] = None):
+    """
+    This disables various destructive functions and prevents the generated code
+    from interfering with the test (e.g. fork bomb, killing other processes,
+    removing filesystem files, etc.)
+
+    WARNING
+    This function is NOT a security sandbox. Untrusted code, including, model-
+    generated code, should not be blindly executed outside of one. See the
+    Codex paper for more information about OpenAI's code sandbox, and proceed
+    with caution.
+    """
+
+    if maximum_memory_bytes is not None:
+        import resource
+
+        resource.setrlimit(resource.RLIMIT_AS, (maximum_memory_bytes, maximum_memory_bytes))
+        resource.setrlimit(resource.RLIMIT_DATA, (maximum_memory_bytes, maximum_memory_bytes))
+        if not platform.uname().system == 'Darwin':
+            resource.setrlimit(resource.RLIMIT_STACK, (maximum_memory_bytes, maximum_memory_bytes))
+
+    faulthandler.disable()
+
+    import builtins
+
+    builtins.exit = None
+    builtins.quit = None
+
+    import os
+
+    os.environ['OMP_NUM_THREADS'] = '1'
+
+    os.kill = None
+    os.system = None
+    os.putenv = None
+    os.remove = None
+    os.removedirs = None
+    os.rmdir = None
+    os.fchdir = None
+    os.setuid = None
+    os.fork = None
+    os.forkpty = None
+    os.killpg = None
+    os.rename = None
+    os.renames = None
+    os.truncate = None
+    os.replace = None
+    os.unlink = None
+    os.fchmod = None
+    os.fchown = None
+    os.chmod = None
+    os.chown = None
+    os.chroot = None
+    os.fchdir = None
+    os.lchflags = None
+    os.lchmod = None
+    os.lchown = None
+    os.getcwd = None
+    os.chdir = None
+
+    import shutil
+
+    shutil.rmtree = None
+    shutil.move = None
+    shutil.chown = None
+
+    import subprocess
+
+    subprocess.Popen = None  # type: ignore
+
+    __builtins__['help'] = None
+
+    import sys
+
+    sys.modules['ipdb'] = None
+    sys.modules['joblib'] = None
+    sys.modules['resource'] = None
+    sys.modules['psutil'] = None
+    sys.modules['tkinter'] = None

evalscope/benchmarks/ifeval/instructions_util.py

@@ -14,7 +14,6 @@
 """Utility library of instructions."""
 
 import functools
-import immutabledict
 import nltk
 import os
 import random
@@ -1551,7 +1550,7 @@ WORD_LIST = [
 ]  # pylint: disable=line-too-long
 
 # ISO 639-1 codes to language names.
-LANGUAGE_CODES = immutabledict.immutabledict({
+LANGUAGE_CODES = {
     'en': 'English',
     'es': 'Spanish',
     'pt': 'Portuguese',
@@ -1582,7 +1581,7 @@ LANGUAGE_CODES = immutabledict.immutabledict({
     'pa': 'Punjabi',
     'ml': 'Malayalam',
     'fi': 'Finnish',
-})
+}
 
 _ALPHABETS = '([A-Za-z])'
 _PREFIXES = '(Mr|St|Mrs|Ms|Dr)[.]'

evalscope/benchmarks/image_edit/gedit/gedit_adapter.py

@@ -0,0 +1,138 @@
+# Copyright (c) Alibaba, Inc. and its affiliates.
+import copy
+import os
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, ImageEditAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessage, ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.metric.scorer import Score
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import FileConstants, Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+SUBSET_LIST = [
+    'background_change', 'color_alter', 'material_alter', 'motion_change', 'ps_human', 'style_change', 'subject-add',
+    'subject-remove', 'subject-replace', 'text_change', 'tone_transfer'
+]
+
+LANGUAGE_LIST = ['en', 'cn']
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='gedit',
+        pretty_name='GEdit-Bench',
+        dataset_id='stepfun-ai/GEdit-Bench',
+        description='GEdit-Bench Image Editing Benchmark, grounded in real-world '
+        'usages is developed to support more authentic and '
+        'comprehensive evaluation of image editing models.',
+        tags=[Tags.IMAGE_EDITING],
+        subset_list=SUBSET_LIST,
+        metric_list=['Semantic Consistency', 'Perceptual Similarity'],
+        few_shot_num=0,
+        train_split=None,
+        eval_split='train',
+        extra_params={'language': f'# language of the instruction, choose from {LANGUAGE_LIST}, default to `en`'}
+    )
+)
+class GEditAdapter(ImageEditAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.language = self.extra_params.get('language', 'en')
+        if self.language not in LANGUAGE_LIST:
+            logger.warning(f"Invalid language '{self.language}', fallback to 'en'")
+            self.language = 'en'
+        self.reformat_subset = True
+        self._use_llm_judge = True
+
+        self.load_prompt()
+
+    def load_prompt(self):
+        from . import vie_prompts
+
+        self.context = vie_prompts._context_no_delimit
+        self.SC_prompt = '\n'.join([
+            self.context, vie_prompts._prompts_0shot_two_image_edit_rule, vie_prompts._prompts_0shot_tie_rule_SC
+        ])
+        self.PQ_prompt = '\n'.join([self.context, vie_prompts._prompts_0shot_rule_PQ])
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+        record = copy.deepcopy(record)
+
+        # Process instruction and image
+        instruction = record['instruction']
+        image_bytes = record['input_image']['bytes']
+        input_image = bytes_to_base64(image_bytes, format='png', add_header=True)
+        record['input_image'] = input_image
+        record[FileConstants.ID] = record['key']
+        del record['input_image_raw']
+
+        text_content = ContentText(text=instruction)
+        image_content = ContentImage(image=input_image)
+
+        messages: List[ChatMessage] = [
+            ChatMessageUser(content=[text_content, image_content]),
+        ]
+
+        return Sample(input=messages, subset_key=record['task_type'], metadata=record)
+
+    def sample_filter(self, sample: Sample) -> bool:
+        language = sample.metadata.get('instruction_language', 'en')
+        return super().sample_filter(sample) and language == self.language
+
+    def llm_match_score(self, original_prediction, filtered_prediction, reference, task_state: TaskState) -> Score:
+        import math
+
+        from .utils import mllm_output_to_dict
+
+        metadata = task_state.metadata
+        text_prompt = metadata['instruction']
+        input_image = metadata['input_image']  # base64 image
+        edited_image = metadata[FileConstants.IMAGE_PATH]  # local image path
+        _SC_prompt = self.SC_prompt.replace('<instruction>', text_prompt)
+
+        # Initialize the score object with prediction details
+        score = Score(
+            extracted_prediction=edited_image,
+            prediction=edited_image,
+        )
+
+        # Build prompts
+        SC_prompt_final = [
+            ChatMessageUser(
+                content=[
+                    ContentImage(image=input_image),
+                    ContentImage(image=edited_image),
+                    ContentText(text=_SC_prompt)
+                ]
+            )
+        ]
+        PQ_prompt_final = [
+            ChatMessageUser(content=[ContentImage(image=edited_image),
+                                     ContentText(text=self.PQ_prompt)])
+        ]
+
+        guess_if_cannot_parse = True
+        result_SC = self.llm_judge.judge(messages=SC_prompt_final)
+        result_PQ = self.llm_judge.judge(messages=PQ_prompt_final)
+        SC_dict = mllm_output_to_dict(result_SC, give_up_parsing=guess_if_cannot_parse)
+        PQ_dict = mllm_output_to_dict(result_PQ, give_up_parsing=guess_if_cannot_parse)
+
+        SC_score = min(SC_dict['score'])
+        PQ_score = min(PQ_dict['score'])
+        O_score = math.sqrt(SC_score * PQ_score)
+
+        score.value = {'Semantic Consistency': SC_score, 'Perceptual Quality': PQ_score, 'Overall': O_score}
+        score.main_score_name = 'Overall'
+        score.metadata = {
+            'SC_dict': SC_dict,
+            'PQ_dict': PQ_dict,
+        }
+        return score