crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/vision_language/real_world_qa_scenario.py

@@ -0,0 +1,57 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.images_utils import generate_hash
+
+
+class RealWorldQAScenario(Scenario):
+    """
+    RealWorldQA is a benchmark designed for real-world understanding. The dataset consists of anonymized
+    images taken from vehicles, in addition to other real-world images.
+
+    Blog post: https://x.ai/blog/grok-1.5v
+    Website: https://huggingface.co/datasets/xai-org/RealworldQA
+    """
+
+    HUGGINGFACE_DATASET_NAME: str = "xai-org/RealworldQA"
+
+    name = "real_world_qa"
+    description = (
+        "A benchmark designed to to evaluate real-world spatial understanding capabilities of multimodal models "
+        "([xAI, 2024](https://x.ai/blog/grok-1.5v))."
+    )
+    tags = ["vision-language", "knowledge", "reasoning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        for row in tqdm(load_dataset(self.HUGGINGFACE_DATASET_NAME, split=TEST_SPLIT, cache_dir=output_path)):
+            # Save the image to disk
+            image = row["image"]
+            image_file_name: str = generate_hash(image) + ".jpg"
+            local_image_path: str = os.path.join(output_path, image_file_name)
+            if not os.path.exists(local_image_path):
+                image.save(local_image_path)
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type="image/jpeg"),
+                MediaObject(text=row["question"], content_type="text/plain"),
+            ]
+            references: List[Reference] = [Reference(output=Output(text=row["answer"]), tags=[CORRECT_TAG])]
+            instances.append(
+                Instance(Input(multimedia_content=MultimediaObject(content)), references=references, split=TEST_SPLIT)
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/seed_bench_scenario.py

@@ -0,0 +1,131 @@
+import os.path
+from typing import Dict, List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+
+
+class SEEDBenchScenario(Scenario):
+    """
+    SEED-Bench: Benchmarking Multimodal LLMs with Generative Comprehension
+
+    Based on powerful Large Language Models (LLMs), recent generative Multimodal
+    Large Language Models (MLLMs) have gained prominence as a pivotal research area.
+    In Seed-Bench, we address the evaluation of generative comprehension in MLLMs
+    as a preliminary step towards a comprehensive assessment of generative models.
+    SEED-Bench consists of 19K multiple choice questions with accurate human annotations
+    (x 6 larger than existing benchmarks), which spans 12 evaluation dimensions
+    including the comprehension of both the image and video modality. We select 9
+    evaluation aspects that take image as the input. In the benchmark,
+    Multiple-choice questions with groundtruth options derived from human
+    annotation enables an objective and efficient assessment of model performance,
+    eliminating the need for human or GPT intervention during evaluation. We employ
+    the multiple-choice metric for evaluating the performance of models.
+
+    @article{li2023seed,
+        title={Seed-bench: Benchmarking multimodal llms with generative comprehension},
+        author={Li, Bohao and Wang, Rui and Wang, Guangzhi and Ge, Yuying and Ge, Yixiao and Shan, Ying},
+        journal={arXiv preprint arXiv:2307.16125},
+        year={2023}
+    }
+
+    Paper: https://arxiv.org/abs/2307.16125
+    """
+
+    SEED_BENCH_HUGGINGFACE_DATASET_NAME: str = "lmms-lab/SEED-Bench"
+
+    SUBJECTS: Dict[str, int] = {
+        "scene-understanding": 1,
+        "instance-identity": 2,
+        "instance-attributes": 3,
+        "instance-location": 4,
+        "instances-counting": 5,
+        "spatial-relation": 6,
+        "instance-interaction": 7,
+        "visual-reasoning": 8,
+        "text-understanding": 9,
+    }
+
+    name = "seed_bench"
+    description = (
+        "Evaluate multimodal models on 9 evaluation aspects " "([Li et al., 2023](https://arxiv.org/abs/2307.16125))."
+    )
+    tags = ["vision-language"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+    def get_subject_name(self, subject_name: str) -> str:
+        return "-".join(subject_name.lower().split())
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        # There is only the test split in Unicorn benchmark
+        instances: List[Instance] = []
+        # Process the test set
+        # Two open-ended generation instances and
+        # one multi-choice generation instance per row
+        for row in tqdm(
+            load_dataset(
+                self.SEED_BENCH_HUGGINGFACE_DATASET_NAME,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            question_type_key: str = self.get_subject_name(self._subject)
+            if row["question_type_id"] != self.SUBJECTS[question_type_key]:
+                continue
+            question_id: str = row["question_id"]
+            # Download the image
+            # Save the image locally
+            image_path: str = os.path.join(images_path, f"{question_id}.png")
+            if not os.path.exists(image_path):
+                # some images are CMYK mode, convert to RGB.
+                row["image"][0].convert("RGB").save(image_path, "PNG", optimize=True)
+
+            # Add the references
+            references: List[Reference] = []
+            question: str = row["question"]
+            answer: str
+            content: List[MediaObject]
+            options: List[str] = [row["choice_a"], row["choice_b"], row["choice_c"], row["choice_d"]]
+            answer = row["answer"].strip()
+            # The given correct answer is a letter, but we need an index
+            correct_answer_index: int = ord(answer) - ord("A")
+            # The options are originally appended to the question
+
+            for i, option in enumerate(options):
+                reference: Reference
+                is_correct: bool = i == correct_answer_index
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if is_correct else [])
+                references.append(reference)
+
+            content = [
+                MediaObject(location=image_path, content_type="image/png"),
+                MediaObject(text=question, content_type="text/plain"),
+            ]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=references,
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/unicorn_scenario.py

@@ -0,0 +1,108 @@
+import os.path
+from typing import Dict, List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class UnicornScenario(Scenario):
+    """
+    How Many Unicorns are in this Image? A Safety Evaluation Benchmark of Vision LLMs
+
+    We shift our focus from evaluating standard performance to introducing a comprehensive safety evaluation
+    suite Unicorn, covering both out-of-distribution (OOD) generalization and adversarial robustness. For the OOD
+    evaluation, we present two novel VQA datasets --- OODCV-VQA and Sketchy-VQA, each with one variant, designed
+    to test model performance under challenging conditions. In the OOD scenario, questions are matched with
+    boolean or numerical answers, and we use exact match metrics for evaluation. When comparing OOD Sketchy-VQA
+    with its synthesized in-distribution counterpart, we found an average model output F1 drop of 8.9%,
+    highlighting the challenging nature of the OOD scenario in the Unicorn benchmark.
+
+    @article{tu2023unicorns,
+    title={How Many Unicorns Are in This Image? A Safety Evaluation Benchmark for Vision LLMs},
+    author={Tu, Haoqin and Cui, Chenhang and Wang, Zijun and Zhou, Yiyang and Zhao, Bingchen and Han,
+    Junlin and Zhou, Wangchunshu and Yao, Huaxiu and Xie, Cihang},
+    journal={arXiv preprint arXiv:2311.16101},
+    year={2023}
+    }
+
+    Paper: https://arxiv.org/abs/2311.16101
+    """
+
+    UNICORN_HUGGINGFACE_DATASET_URL: str = "https://huggingface.co/datasets/PahaII/unicorn/resolve/main"
+
+    IMAGE_URL: str = "https://huggingface.co/datasets/PahaII/unicorn/resolve/main/images/{image_path}?download=true"
+
+    SUBJECTS: List[str] = ["OODCV-VQA", "OODCV-Counterfactual", "Sketchy-VQA", "Sketchy-Challenging"]
+
+    IMG_TYPE: Dict[str, str] = {
+        "OODCV-VQA": "jpeg",
+        "OODCV-Counterfactual": "jpeg",
+        "Sketchy-VQA": "png",
+        "Sketchy-Challenging": "png",
+    }
+
+    name = "unicorn"
+    description = (
+        "Evaluate multimodal models on two out-of-distribution scenarios with four subjects "
+        "([Tu et al., 2023](https://arxiv.org/abs/2311.16101))."
+    )
+    tags = ["vision-language"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+        self._image_type: str = self.IMG_TYPE[self._subject]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        # There is only the test split in Unicorn benchmark
+        instances: List[Instance] = []
+        question_data_files = {TEST_SPLIT: f"{self.UNICORN_HUGGINGFACE_DATASET_URL}/{self._subject}.json"}
+
+        # Process the test set
+        for row in tqdm(
+            load_dataset(
+                "json",
+                data_files=question_data_files,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            # Download the image
+            image_path: str = row["image_path"]
+            local_image_path: str = os.path.join(output_path, image_path)
+            ensure_file_downloaded(
+                source_url=self.IMAGE_URL.format(image_path=image_path),
+                target_path=local_image_path,
+                unpack=False,
+            )
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type=f"image/{self._image_type}"),
+                MediaObject(text=row["question"], content_type="text/plain"),
+            ]
+            answer: str = row["answer"]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/vibe_eval_scenario.py

@@ -0,0 +1,98 @@
+import os.path
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+
+
+class VibeEvalScenario(Scenario):
+    """
+    Vibe-Eval: A hard evaluation suite for measuring progress of multimodal language models
+
+    We introduce Vibe-Eval: a new open benchmark and framework for evaluating multimodal chat
+    models. Vibe-Eval consists of 269 visual understanding prompts, including 100 of hard
+    difficulty, complete with gold-standard responses authored by experts. Vibe-Eval is
+    open-ended and challenging with dual objectives: (i) vibe checking multimodal chat models
+    for day-to-day tasks and (ii) rigorously testing and probing the capabilities of present
+    frontier models. Notably, our hard set contains >50% questions that all frontier models
+    answer incorrectly. We also discuss trade-offs between human and automatic evaluation,
+    and show that automatic model evaluation using Reka Core roughly correlates to human judgment.
+
+    @article{padlewski2024vibe,
+    title={Vibe-Eval: A hard evaluation suite for measuring progress of multimodal language models},
+    author={Padlewski, Piotr and Bain, Max and Henderson, Matthew and Zhu, Zhongkai
+    and Relan, Nishant and Pham, Hai and Ong, Donovan and Aleksiev, Kaloyan and Ormazabal, Aitor
+    and Phua, Samuel and others},
+    journal={arXiv preprint arXiv:2405.02287},
+    year={2024}
+    }
+
+    Paper: https://arxiv.org/abs/2405.02287
+    """
+
+    VIBE_EVAL_HUGGINGFACE_DATASET_NAME: str = "RekaAI/VibeEval"
+
+    SUBJECTS: List[str] = [
+        "difficulty-hard",
+        "difficulty-normal",
+    ]
+
+    name = "vibe_eval"
+    description = (
+        "Evaluate multimodal models on day-to-day tasks "
+        "([Padlewski et al., 2024](https://arxiv.org/abs/2405.02287))."
+    )
+    tags = ["vision-language", "knowledge", "reasoning"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        instances: List[Instance] = []
+        # Process the test set
+        for row in tqdm(
+            load_dataset(
+                self.VIBE_EVAL_HUGGINGFACE_DATASET_NAME,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            if row["category"] != self._subject:
+                continue
+            example_id: str = row["example_id"].replace("/", "-")
+            # Save the image locally
+            local_image_path: str = os.path.join(images_path, f"{example_id}.png")
+            if not os.path.exists(local_image_path):
+                row["image"].convert("RGB").save(local_image_path, "PNG", optimize=True)
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type="image/png"),
+                MediaObject(text=row["prompt"], content_type="text/plain"),
+            ]
+            answer: str = row["reference"]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/viz_wiz_scenario.py

@@ -13,13 +13,13 @@ from helm.benchmark.scenarios.scenario import (
     Scenario,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+from helm.common.general import ensure_file_downloaded
 
 
 class VizWizScenario(Scenario):
     """
-    VizWiz is a real-world visual question answering dataset consisting of questions
-    asked by people who are blind. It originates from a natural visual question answering
+    VizWiz is a real-world visual question answering dataset consisting of questions asked by people who are
+    visually impaired. It originates from a natural visual question answering
     setting where blind people each took an image and recorded a spoken question about it,
     together with 10 crowdsourced answers per visual question.
 
@@ -53,14 +53,13 @@ class VizWizScenario(Scenario):
     name = "viz_wiz"
     description = (
         "Real-world VQA dataset consisting of questions asked by "
-        "people who are blind ([paper](https://arxiv.org/abs/1802.08218))."
+        "people who are blind ([Gurari et al., 2018](https://arxiv.org/abs/1802.08218))."
     )
     tags = ["vision-language", "visual question answering"]
 
     def get_instances(self, output_path: str) -> List[Instance]:
         # Download the questions and annotations
         annotations_path: str = os.path.join(output_path, "annotations")
-        ensure_directory_exists(annotations_path)
         ensure_file_downloaded(
             source_url=self.ANNOTATIONS_URL,
             target_path=annotations_path,

helm/benchmark/scenarios/vision_language/vqa_rad_scenario.py

@@ -0,0 +1,88 @@
+import os
+from typing import List
+
+from datasets import DatasetDict, load_dataset
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    Input,
+    Instance,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.general import ensure_directory_exists
+from helm.common.media_object import MediaObject, MultimediaObject
+
+
+class VQARadScenario(Scenario):
+    """
+    VQARad scenario: Processes a visual question answering dataset with radiology images.
+
+    Each record in the dataset has:
+    - image
+    - question
+    - answer
+
+    The output is formatted as:
+    "Answer: <answer>"
+    """
+
+    HUGGING_FACE_DATASET_PATH: str = "flaviagiammarino/vqa-rad"
+
+    name = "vqa_rad"
+    description = "Visual question answering with radiology images."
+    tags = [
+        "vision-language",
+        "visual question answering",
+        "reasoning",
+        "medical",
+        "radiology",
+    ]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset: DatasetDict = load_dataset(self.HUGGING_FACE_DATASET_PATH)
+
+        splits = {TRAIN_SPLIT: "train", TEST_SPLIT: "test"}
+        instances: List[Instance] = []
+        # Iterate over the splits
+        for (
+            helm_split_name,
+            dataset_split_name,
+        ) in splits.items():
+            split_path: str = os.path.join(output_path, dataset_split_name)
+            ensure_directory_exists(split_path)
+
+            split_data = dataset[dataset_split_name]
+
+            for index, example in enumerate(split_data):
+                question = example["question"]
+                image = example["image"]
+                answer = example["answer"]
+
+                # Convert PIL image to MediaObject
+                image_path = os.path.join(split_path, f"{index}.jpg")
+                image.save(image_path)
+
+                content = [
+                    MediaObject(location=image_path, content_type="image/jpeg"),
+                    MediaObject(text=question, content_type="text/plain"),
+                ]
+
+                # Format the final answer
+                instances.append(
+                    Instance(
+                        input=Input(multimedia_content=MultimediaObject(content)),
+                        references=[
+                            Reference(
+                                Output(text=answer),
+                                tags=[CORRECT_TAG],
+                            )
+                        ],
+                        split=helm_split_name,
+                    )
+                )
+
+        return instances

helm/benchmark/scenarios/vision_language/vqa_scenario.py

@@ -54,8 +54,10 @@ class VQAScenario(Scenario):
         TEST_SPLIT: "http://images.cocodataset.org/zips/test2015.zip",
     }
 
-    name = "visual_question_answering"
-    description = "Open-ended questions about images ([paper](https://arxiv.org/abs/1612.00837))."
+    name = "vqa"
+    description = (
+        "Open-ended questions about real-world images " "([Goyal et al., 2017](https://arxiv.org/abs/1612.00837))."
+    )
     tags = ["vision-language", "visual question answering"]
 
     def get_instances(self, output_path: str) -> List[Instance]:
@@ -111,11 +113,13 @@ class VQAScenario(Scenario):
                     MediaObject(location=image_path, content_type="image/jpeg"),
                     MediaObject(text=question_json["question"], content_type="text/plain"),
                 ]
-
                 instances.append(
                     Instance(
                         Input(multimedia_content=MultimediaObject(content)),
-                        references=[Reference(Output(text=answers_json["multiple_choice_answer"]), tags=[CORRECT_TAG])],
+                        references=[
+                            Reference(Output(text=answer_json["answer"]), tags=[CORRECT_TAG])
+                            for answer_json in answers_json["answers"]
+                        ],
                         split=split,
                     )
                 )

helm/benchmark/scenarios/wikifact_scenario.py

@@ -2,9 +2,21 @@ import os
 from typing import List, Dict
 import json
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded, flatten_list
 from helm.common.hierarchical_logger import hlog
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 PID_TO_NAME = {
     "P136": "genre",
@@ -173,3 +185,21 @@ class WIKIFactScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="wikifact",
+            display_name="WikiFact",
+            description="Scenario introduced in this work, inspired by [Petroni et al. "
+            "(2019)](https://aclanthology.org/D19-1250/), to more extensively test factual "
+            "knowledge.",
+            taxonomy=TaxonomyInfo(
+                task="knowledge base completion",
+                what="entity-relation-entity triples in natural language form",
+                when="?",
+                who="automatically generated from templates",
+                language="structured English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/wikitext_103_scenario.py

@@ -4,7 +4,7 @@ from typing import List
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
 
 
 class Wikitext103Scenario(Scenario):