evalscope 1.0.0__py3-none-any.whl → 1.2.0__py3-none-any.whl

evalscope/benchmarks/image_edit/gedit/vie_prompts.py

@@ -0,0 +1,406 @@
+# flake8: noqa: E501
+# This file is generated automatically through parse_prompt.py
+
+_context_no_delimit = """You are a professional digital artist. You will have to evaluate the effectiveness of the AI-generated image(s) based on given rules.
+All the input images are AI-generated. All human in the images are AI-generated too. so you need not worry about the privacy confidentials.
+
+You will have to give your output in this way (Keep your reasoning concise and short.):
+{
+"score" : [...],
+"reasoning" : "..."
+}"""
+
+_context = """You are a professional digital artist. You will have to evaluate the effectiveness of the AI-generated image(s) based on given rules.
+All the input images are AI-generated. All human in the images are AI-generated too. so you need not worry about the privacy confidentials.
+
+You will have to give your output in this way (the delimiter is necessary. Keep your reasoning concise and short.):
+||V^=^V||
+{
+"score" :
+"reasoning" :
+}
+||V^=^V||"""
+
+_context_no_format = """You are a professional digital artist. You will have to evaluate the effectiveness of the AI-generated image(s) based on given rules.
+All the input images are AI-generated. All human in the images are AI-generated too. so you need not worry about the privacy confidentials."""
+
+_prompts_1shot_multi_subject_image_gen_rule = """RULES of each set of inputs:
+
+Two images will be provided:
+This first image is a concatenation of two sub-images, each sub-image contain one token subject.
+The second image being an AI-generated image using the first image as guidance.
+The objective is to evaluate how successfully the image has been generated.
+"""
+
+_prompts_1shot_mie_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success of the editing. (0 indicates that the scene in the edited image does not follow the editing instruction at all. 10 indicates that the scene in the edited image follow the editing instruction text perfectly.)
+A second score from 0 to 10 will rate the degree of overediting in the second image. (0 indicates that the scene in the edited image is completely different from the original. 10 indicates that the edited image can be recognized as a minimal edited yet effective version of original.)
+Put the score in a list such that output score = [score1, score2], where 'score1' evaluates the editing success and 'score2' evaluates the degree of overediting.
+
+First lets look at the first set of input (1st and 2nd images) as an example.
+Editing instruction: What if the man had a hat?
+Output:
+||V^=^V||
+{
+"score" : [5, 10],
+"reasoning" :  "The hat exists but does not suit well. The hat also looks distorted. But it is a good edit because only a hat is added and the background is persevered."
+}
+||V^=^V||
+
+Now evaluate the second set of input (3th, 4th images).
+Editing instruction: <instruction>
+"""
+
+_prompts_1shot_msdig_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success in following the prompt.
+(0 indicates that the second image does not follow the prompt at all. 10 indicates the second image follows the prompt perfectly.)
+A second score from 0 to 10 will rate how well the subject in the generated image resemble to the token subject in the first sub-image.
+(0 indicates that the subject in the second image does not look like the token subject in the first sub-image at all. 10 indicates the subject in the second image look exactly alike the token subject in the first sub-image.)
+A third score from 0 to 10 will rate how well the subject in the generated image resemble to the token subject in the second sub-image.
+(0 indicates that the subject in the second image does not look like the token subject in the second sub-image at all. 10 indicates the subject in the second image look exactly alike the token subject in the second sub-image.)
+Put the score in a list such that output score = [score1, score2, score3], where 'score1' evaluates the prompt and 'score2' evaluates the resemblance for the first sub-image, and 'score3' evaluates the resemblance for the second sub-image.
+
+First lets look at the first set of input (1st and 2nd images) as an example.
+Text Prompt: A digital illustration of a cat beside a wooden pot
+Output:
+||V^=^V||
+{
+"score" : [5, 5, 10],
+"reasoning" :  "The cat is not beside the wooden pot. The pot looks partially resemble to the subject pot. The cat looks highly resemble to the subject cat."
+}
+||V^=^V||
+
+Now evaluate the second set of input (3th, 4th images).
+Text Prompt: <prompt>"""
+
+_prompts_1shot_t2i_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success in following the prompt.
+(0 indicates that the AI generated image does not follow the prompt at all. 10 indicates the AI generated image follows the prompt perfectly.)
+
+Put the score in a list such that output score = [score].
+
+First lets look at the first set of input (1st image) as an example.
+Text Prompt: A pink and a white frisbee are on the ground.
+Output:
+||V^=^V||
+{
+"score" : [5],
+"reasoning" :  "White frisbee not present in the image."
+}
+||V^=^V||
+
+Now evaluate the second set of input (2nd image).
+Text Prompt: <prompt>
+"""
+
+_prompts_1shot_tie_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success of the editing. (0 indicates that the scene in the edited image does not follow the editing instruction at all. 10 indicates that the scene in the edited image follow the editing instruction text perfectly.)
+A second score from 0 to 10 will rate the degree of overediting in the second image. (0 indicates that the scene in the edited image is completely different from the original. 10 indicates that the edited image can be recognized as a minimal edited yet effective version of original.)
+Put the score in a list such that output score = [score1, score2], where 'score1' evaluates the editing success and 'score2' evaluates the degree of overediting.
+
+First lets look at the first set of input (1st and 2nd images) as an example.
+Editing instruction: What if the man had a hat?
+Output:
+||V^=^V||
+{
+"score" : [5, 10],
+"reasoning" :  "The hat exists but does not suit well. The hat also looks distorted. But it is a good edit because only a hat is added and the background is persevered."
+}
+||V^=^V||
+
+Now evaluate the second set of input (3th, 4th images).
+Editing instruction: <instruction>
+"""
+
+_prompts_1shot_sdie_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will rate how well the subject in the generated image resemble to the token subject in the second image.
+(0 indicates that the subject in the third image does not look like the token subject at all. 10 indicates the subject in the third image look exactly alike the token subject.)
+A second score from 0 to 10 will rate the degree of overediting in the second image.
+(0 indicates that the scene in the edited image is completely different from the first image. 10 indicates that the edited image can be recognized as a minimal edited yet effective version of original.)
+Put the score in a list such that output score = [score1, score2], where 'score1' evaluates the resemblance and 'score2' evaluates the degree of overediting.
+
+First lets look at the first set of input (1st, 2nd and 3rd images) as an example.
+Subject: <subject>
+Output:
+||V^=^V||
+{
+"score" : [5, 10],
+"reasoning" :  "The monster toy looks partially resemble to the token subject. The edit is minimal."
+}
+||V^=^V||
+
+Now evaluate the second set of input (4th, 5th, and 6th images).
+Subject: <subject>
+"""
+
+_prompts_1shot_one_image_gen_rule = """RULES of each set of inputs:
+
+One image will be provided; The image is an AI-generated image.
+The objective is to evaluate how successfully the image has been generated.
+"""
+
+_prompts_1shot_sdig_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success in following the prompt.
+(0 indicates that the second image does not follow the prompt at all. 10 indicates the second image follows the prompt perfectly.)
+A second score from 0 to 10 will rate how well the subject in the generated image resemble to the token subject in the first image.
+(0 indicates that the subject in the second image does not look like the token subject at all. 10 indicates the subject in the second image look exactly alike the token subject.)
+Put the score in a list such that output score = [score1, score2], where 'score1' evaluates the prompt and 'score2' evaluates the resemblance.
+
+First lets look at the first set of input (1st and 2nd images) as an example.
+Text Prompt: a red cartoon figure eating a banana
+Output:
+||V^=^V||
+{
+"score" : [10, 5],
+"reasoning" :  "The red cartoon figure is eating a banana. The red cartoon figure looks partially resemble to the subject."
+}
+||V^=^V||
+
+Now evaluate the second set of input (3th, 4th images).
+Text Prompt: <prompt>
+"""
+
+_prompts_1shot_rule_PQ = """RULES of each set of inputs:
+
+One image will be provided; The image is an AI-generated image.
+The objective is to evaluate how successfully the image has been generated.
+
+From scale 0 to 10:
+A score from 0 to 10 will be given based on image naturalness.
+(
+    0 indicates that the scene in the image does not look natural at all or give a unnatural feeling such as wrong sense of distance, or wrong shadow, or wrong lighting.
+    10 indicates that the image looks natural.
+)
+A second score from 0 to 10 will rate the image artifacts.
+(
+    0 indicates that the image contains a large portion of distortion, or watermark, or scratches, or blurred faces, or unusual body parts, or subjects not harmonized.
+    10 indicates the image has no artifacts.
+)
+Put the score in a list such that output score = [naturalness, artifacts]
+
+
+First lets look at the first set of input (1st image) as an example.
+Output:
+||V^=^V||
+{
+"score" : [5, 5],
+"reasoning" :  "The image gives an unnatural feeling on hands of the girl. There is also minor distortion on the eyes of the girl."
+}
+||V^=^V||
+
+Now evaluate the second set of input (2nd image).
+
+"""
+
+_prompts_1shot_subject_image_gen_rule = """RULES of each set of inputs:
+
+Two images will be provided: The first being a token subject image and the second being an AI-generated image using the first image as guidance.
+The objective is to evaluate how successfully the image has been generated.
+"""
+
+_prompts_1shot_cig_rule_SC = """
+From scale 0 to 10:
+A score from 0 to 10 will be given based on the success in following the prompt.
+(0 indicates that the second image does not follow the prompt at all. 10 indicates the second image follows the prompt perfectly.)
+A second score from 0 to 10 will rate how well the generated image is following the guidance image.
+(0 indicates that the second image is not following the guidance at all. 10 indicates that second image is following the guidance image.)
+Put the score in a list such that output score = [score1, score2], where 'score1' evaluates the prompt and 'score2' evaluates the guidance.
+
+First lets look at the first set of input (1st and 2nd images) as an example.
+Text Prompt: the bridge is red, Golden Gate Bridge in San Francisco, USA
+Output:
+||V^=^V||
+{
+"score" : [5, 5],
+"reasoning" :  "The bridge is red. But half of the bridge is gone."
+}
+||V^=^V||
+
+Now evaluate the second set of input (3th, 4th images).
+Text Prompt: <prompt>
+"""
+
+_prompts_1shot_two_image_edit_rule = """RULES of each set of inputs:
+
+Two images will be provided: The first being the original AI-generated image and the second being an edited version of the first.
+The objective is to evaluate how successfully the editing instruction has been executed in the second image.
+
+Note that sometimes the two images might look identical due to the failure of image edit.
+"""
+
+_prompts_1shot_subject_image_edit_rule = """RULES of each set of inputs:
+
+Three images will be provided:
+The first image is a input image to be edited.
+The second image is a token subject image.
+The third image is an AI-edited image from the first image. it should contain a subject that looks alike the subject in second image.
+The objective is to evaluate how successfully the image has been edited.
+"""
+
+_prompts_1shot_control_image_gen_rule = """RULES of each set of inputs:
+
+Two images will be provided: The first being a processed image (e.g. Canny edges, openpose, grayscale etc.) and the second being an AI-generated image using the first image as guidance.
+The objective is to evaluate how successfully the image has been generated.
+"""
+
+_prompts_0shot_two_image_edit_rule = """RULES:
+
+Two images will be provided: The first being the original AI-generated image and the second being an edited version of the first.
+The objective is to evaluate how successfully the editing instruction has been executed in the second image.
+
+Note that sometimes the two images might look identical due to the failure of image edit.
+"""
+
+_prompts_0shot_one_video_gen_rule = """RULES:
+
+The images are extracted from a AI-generated video according to the text prompt.
+The objective is to evaluate how successfully the video has been generated.
+"""
+
+_prompts_0shot_t2v_rule_PQ = """RULES:
+
+The image frames are AI-generated.
+The objective is to evaluate how successfully the image frames has been generated.
+
+From scale 0 to 10:
+A score from 0 to 10 will be given based on the image frames naturalness.
+(
+    0 indicates that the scene in the image frames does not look natural at all or give a unnatural feeling such as wrong sense of distance, or wrong shadow, or wrong lighting.
+    10 indicates that the image frames looks natural.
+)
+A second score from 0 to 10 will rate the image frames artifacts.
+(
+    0 indicates that the image frames contains a large portion of distortion, or watermark, or scratches, or blurred faces, or unusual body parts, or subjects not harmonized.
+    10 indicates the image frames has no artifacts.
+)
+Put the score in a list such that output score = [naturalness, artifacts]
+"""
+
+_prompts_0shot_msdig_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success in following the prompt.
+(0 indicates that the second image does not follow the prompt at all. 10 indicates the second image follows the prompt perfectly.)
+A second score from 0 to 10 will rate how well the subject in the generated image resemble to the token subject in the first sub-image.
+(0 indicates that the subject in the second image does not look like the token subject in the first sub-image at all. 10 indicates the subject in the second image look exactly alike the token subject in the first sub-image.)
+A third score from 0 to 10 will rate how well the subject in the generated image resemble to the token subject in the second sub-image.
+(0 indicates that the subject in the second image does not look like the token subject in the second sub-image at all. 10 indicates the subject in the second image look exactly alike the token subject in the second sub-image.)
+Put the score in a list such that output score = [score1, score2, score3], where 'score1' evaluates the prompt and 'score2' evaluates the resemblance for the first sub-image, and 'score3' evaluates the resemblance for the second sub-image.
+
+Text Prompt: <prompt>
+"""
+
+_prompts_0shot_sdie_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will rate how well the subject in the generated image resemble to the token subject in the second image.
+(0 indicates that the subject in the third image does not look like the token subject at all. 10 indicates the subject in the third image look exactly alike the token subject.)
+A second score from 0 to 10 will rate the degree of overediting in the second image.
+(0 indicates that the scene in the edited image is completely different from the first image. 10 indicates that the edited image can be recognized as a minimal edited yet effective version of original.)
+Put the score in a list such that output score = [score1, score2], where 'score1' evaluates the resemblance and 'score2' evaluates the degree of overediting.
+
+Subject: <subject>"""
+
+_prompts_0shot_subject_image_edit_rule = """RULES:
+
+Three images will be provided:
+The first image is a input image to be edited.
+The second image is a token subject image.
+The third image is an AI-edited image from the first image. it should contain a subject that looks alike the subject in second image.
+The objective is to evaluate how successfully the image has been edited.
+"""
+
+_prompts_0shot_mie_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success of the editing. (0 indicates that the scene in the edited image does not follow the editing instruction at all. 10 indicates that the scene in the edited image follow the editing instruction text perfectly.)
+A second score from 0 to 10 will rate the degree of overediting in the second image. (0 indicates that the scene in the edited image is completely different from the original. 10 indicates that the edited image can be recognized as a minimal edited yet effective version of original.)
+Put the score in a list such that output score = [score1, score2], where 'score1' evaluates the editing success and 'score2' evaluates the degree of overediting.
+
+Editing instruction: <instruction>
+"""
+
+_prompts_0shot_sdig_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success in following the prompt.
+(0 indicates that the second image does not follow the prompt at all. 10 indicates the second image follows the prompt perfectly.)
+A second score from 0 to 10 will rate how well the subject in the generated image resemble to the token subject in the first image.
+(0 indicates that the subject in the second image does not look like the token subject at all. 10 indicates the subject in the second image look exactly alike the token subject.)
+Put the score in a list such that output score = [score1, score2], where 'score1' evaluates the prompt and 'score2' evaluates the resemblance.
+
+Text Prompt: <prompt>
+"""
+
+_prompts_0shot_tie_rule_SC = """
+From scale 0 to 10:
+A score from 0 to 10 will be given based on the success of the editing. (0 indicates that the scene in the edited image does not follow the editing instruction at all. 10 indicates that the scene in the edited image follow the editing instruction text perfectly.)
+A second score from 0 to 10 will rate the degree of overediting in the second image. (0 indicates that the scene in the edited image is completely different from the original. 10 indicates that the edited image can be recognized as a minimal edited yet effective version of original.)
+Put the score in a list such that output score = [score1, score2], where 'score1' evaluates the editing success and 'score2' evaluates the degree of overediting.
+
+Editing instruction: <instruction>
+"""
+
+_prompts_0shot_t2i_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success in following the prompt.
+(0 indicates that the AI generated image does not follow the prompt at all. 10 indicates the AI generated image follows the prompt perfectly.)
+
+Put the score in a list such that output score = [score].
+
+Text Prompt: <prompt>
+"""
+
+_prompts_0shot_cig_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success in following the prompt.
+(0 indicates that the second image does not follow the prompt at all. 10 indicates the second image follows the prompt perfectly.)
+A second score from 0 to 10 will rate how well the generated image is following the guidance image.
+(0 indicates that the second image is not following the guidance at all. 10 indicates that second image is following the guidance image.)
+Put the score in a list such that output score = [score1, score2], where 'score1' evaluates the prompt and 'score2' evaluates the guidance.
+
+Text Prompt: <prompt>"""
+
+_prompts_0shot_control_image_gen_rule = """RULES:
+
+Two images will be provided: The first being a processed image (e.g. Canny edges, openpose, grayscale etc.) and the second being an AI-generated image using the first image as guidance.
+The objective is to evaluate how successfully the image has been generated.
+"""
+
+_prompts_0shot_rule_PQ = """RULES:
+
+The image is an AI-generated image.
+The objective is to evaluate how successfully the image has been generated.
+
+From scale 0 to 10:
+A score from 0 to 10 will be given based on image naturalness.
+(
+    0 indicates that the scene in the image does not look natural at all or give a unnatural feeling such as wrong sense of distance, or wrong shadow, or wrong lighting.
+    10 indicates that the image looks natural.
+)
+A second score from 0 to 10 will rate the image artifacts.
+(
+    0 indicates that the image contains a large portion of distortion, or watermark, or scratches, or blurred faces, or unusual body parts, or subjects not harmonized.
+    10 indicates the image has no artifacts.
+)
+Put the score in a list such that output score = [naturalness, artifacts]
+"""
+
+_prompts_0shot_t2v_rule_SC = """From scale 0 to 10:
+A score from 0 to 10 will be given based on the success in following the prompt.
+(0 indicates that the image frames does not follow the prompt at all. 10 indicates the image frames follows the prompt perfectly.)
+
+Put the score in a list such that output score = [score].
+
+Text Prompt: <prompt>
+"""
+
+_prompts_0shot_multi_subject_image_gen_rule = """RULES:
+
+Two images will be provided:
+This first image is a concatenation of two sub-images, each sub-image contain one token subject.
+The second image being an AI-generated image using the first image as guidance.
+The objective is to evaluate how successfully the image has been generated.
+"""
+
+_prompts_0shot_subject_image_gen_rule = """RULES:
+
+Two images will be provided: The first being a token subject image and the second being an AI-generated image using the first image as guidance.
+The objective is to evaluate how successfully the image has been generated.
+"""
+
+_prompts_0shot_one_image_gen_rule = """RULES:
+
+The image is an AI-generated image according to the text prompt.
+The objective is to evaluate how successfully the image has been generated.
+"""

evalscope/benchmarks/infovqa/infovqa_adapter.py

@@ -0,0 +1,66 @@
+import json
+from typing import Any, Dict, List
+
+from evalscope.api.benchmark import BenchmarkMeta, VisionLanguageAdapter
+from evalscope.api.dataset import Sample
+from evalscope.api.evaluator.state import TaskState
+from evalscope.api.messages import ChatMessageUser, Content, ContentImage, ContentText
+from evalscope.api.registry import register_benchmark
+from evalscope.constants import Tags
+from evalscope.utils.io_utils import bytes_to_base64
+from evalscope.utils.logger import get_logger
+
+logger = get_logger()
+
+PROMPT = """Answer the question according to the image using a single word or phrase.
+{question}
+The last line of your response should be of the form "ANSWER: $ANSWER" (without quotes) where $ANSWER is the answer to the question."""  # noqa: E501
+
+
+@register_benchmark(
+    BenchmarkMeta(
+        name='infovqa',
+        pretty_name='InfoVQA',
+        tags=[Tags.MULTI_MODAL, Tags.KNOWLEDGE, Tags.QA],
+        description=
+        'InfoVQA (Information Visual Question Answering) is a benchmark designed to evaluate how well AI models can answer questions based on information-dense images, such as charts, graphs, diagrams, maps, and infographics.',  # noqa: E501
+        dataset_id='lmms-lab/DocVQA',
+        subset_list=['InfographicVQA'],
+        metric_list=['anls'],
+        eval_split='validation',
+        prompt_template=PROMPT,
+    )
+)
+class InfoVQAAdapter(VisionLanguageAdapter):
+
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+        self.add_aggregation_name = False
+
+    def record_to_sample(self, record: Dict[str, Any]) -> Sample:
+
+        input_text = PROMPT.format(question=record['question'])
+        content_list: List[Content] = [ContentText(text=input_text)]
+        image = record.get('image')
+        if image:
+            image_base64 = bytes_to_base64(image['bytes'], format='png', add_header=True)
+            content_list.append(ContentImage(image=image_base64))
+        return Sample(
+            input=[ChatMessageUser(content=content_list)],
+            target=json.dumps(record.get('answers')),  # answers is a list
+            metadata={
+                'questionId': record.get('questionId'),
+                'answer_type': record.get('answer_type'),
+                'image_url': record.get('image_url'),
+                'ocr': record.get('ocr'),
+            }
+        )
+
+    def extract_answer(self, prediction: str, task_state: TaskState) -> str:
+        import re
+
+        pattern = r'ANSWER:\s*(.*)'
+        match = re.search(pattern, prediction)
+        if match:
+            return match.group(1).strip()
+        return prediction.strip()

evalscope/benchmarks/live_code_bench/evaluate_utils.py

@@ -9,6 +9,19 @@ from .pass_k_utils import compute_metrics_from_results
 logger = get_logger()
 
 
+def _temp_run(sample, generation, debug, result, metadata_list, timeout):
+    """Runs a test in a separate process to enforce a timeout.
+    This function is defined at the module's top level to ensure it can be
+    pickled by `multiprocessing.Process`. This is a requirement on platforms
+    like macOS (on Apple Silicon) which use the 'spawn' start method, as
+    nested functions are not picklable.
+    """
+    from .testing_util import run_test
+    res, metadata = run_test(sample, test=generation, debug=debug, timeout=timeout)
+    result.append(res)
+    metadata_list.append(metadata)
+
+
 def codegen_check_correctness(sample, generation, timeout, debug=True):
     """Check correctness of code generation with a global timeout.
 
@@ -16,12 +29,6 @@ def codegen_check_correctness(sample, generation, timeout, debug=True):
     timeouts inside `run_test`
     """
 
-    def _temp_run(sample, generation, debug, result, metadata_list, timeout):
-        from .testing_util import run_test
-        res, metadata = run_test(sample, test=generation, debug=debug, timeout=timeout)
-        result.append(res)
-        metadata_list.append(metadata)
-
     manager = multiprocessing.Manager()
     result = manager.list()
     metadata_list = manager.list()