crfm-helm 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/scenarios/vision_language/bingo_scenario.py

@@ -0,0 +1,103 @@
+import os.path
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class BingoScenario(Scenario):
+    """
+    Holistic Analysis of Hallucination in GPT-4V(ision): Bias and Interference Challenges
+
+    We introduce a new benchmark, namely, the Bias and Interference Challenges in Visual Language Models (Bingo).
+    This benchmark is designed to evaluate and shed light on the two common types of hallucinations in visual
+    language models: bias and interference. Here, bias refers to the model's tendency to hallucinate certain types
+    of responses, possibly due to imbalance in its training data. Interference pertains to scenarios where the
+    judgment of GPT-4V(ision) can be disrupted due to how the text prompt is phrased or how the input image is
+    presented. The benchmark consists of open-ended question-answer pairs, and we employ open-ended generation
+    metrics for evaluation. In the experiment, we identify a notable regional bias, whereby GPT-4V(ision) is
+    better at interpreting Western images or images with English writing compared to images from other countries
+    or containing text in other languages.
+
+
+    @article{cui2023holistic,
+    title={Holistic analysis of hallucination in gpt-4v (ision): Bias and interference challenges},
+    author={Cui, Chenhang and Zhou, Yiyang and Yang, Xinyu and Wu, Shirley and Zhang, Linjun and
+    Zou, James and Yao, Huaxiu},
+    journal={arXiv preprint arXiv:2311.03287},
+    year={2023}
+    }
+
+    Paper: https://arxiv.org/abs/2311.03287
+    """
+
+    BINGO_HUGGINGFACE_DATASET_NAME: str = "PahaII/Bingo"
+
+    IMAGE_URL: str = "https://huggingface.co/datasets/PahaII/Bingo/resolve/main/images/{image_path}?download=true"
+
+    SUBJECTS: List[str] = ["T2I", "I2I", "OCR", "Factual", "Region"]
+
+    name = "bingo"
+    description = (
+        "Evaluate multimodal models on biased and inference-challenging scenarios with five subjects"
+        " ([paper](https://arxiv.org/abs/2311.03287))."
+    )
+    tags = ["vision-language"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        # There is only the test split in Unicorn benchmark
+        instances: List[Instance] = []
+        question_data_files = {TEST_SPLIT: f"{self._subject}.json"}
+
+        # Process the test set
+        for row in tqdm(
+            load_dataset(
+                self.BINGO_HUGGINGFACE_DATASET_NAME,
+                data_files=question_data_files,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            # Download the image
+            image_path: str = row["image_path"]
+            local_image_path: str = os.path.join(output_path, image_path)
+            ensure_file_downloaded(
+                source_url=self.IMAGE_URL.format(image_path=image_path),
+                target_path=local_image_path,
+                unpack=False,
+            )
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type="image/png"),
+                MediaObject(text=row["question"], content_type="text/plain"),
+            ]
+            answer: str = row["answer"]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/hateful_memes_scenario.py

@@ -0,0 +1,92 @@
+import os.path
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    ALL_SPLITS,
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class HatefulMemesScenario(Scenario):
+    """
+    The Hateful Memes Challenge and Dataset is a competition and open source dataset designed to
+    measure progress in multimodal vision-and-language classification.
+
+    In order for AI to become a more effective tool for detecting hate speech, it must be able to understand
+    content the way people do: holistically. When viewing a meme, for example, we don’t think about the words
+    and photo independently of each other; we understand the combined meaning. This is extremely challenging
+    for machines, however, because it means they can’t analyze the text and the image separately. They must
+    combine these different modalities and understand how the meaning changes when they are presented together.
+
+    @article{kiela2020hateful,
+      title={The Hateful Memes Challenge: Detecting Hate Speech in Multimodal Memes},
+      author={Kiela, Douwe and Firooz, Hamed and Mohan, Aravind and Goswami, Vedanuj and Singh,
+      Amanpreet and Ringshia, Pratik and  Testuggine, Davide},
+      journal={arXiv preprint arXiv:2005.04790},
+      year={2020}
+    }
+
+    Paper: https://arxiv.org/abs/2005.04790
+    Website: https://ai.meta.com/tools/hatefulmemes/
+    """
+
+    IMAGE_URL: str = (
+        "https://huggingface.co/datasets/neuralcatcher/hateful_memes/resolve/main/{image_path}?download=true"
+    )
+
+    # This question is from the example on the website
+    QUESTION: str = "Is this meme mean?"
+
+    name = "hateful_memes"
+    description = "Identify if a meme is hateful or not ([paper](https://arxiv.org/abs/2005.04790))."
+    tags = ["vision-language"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        ensure_directory_exists(os.path.join(output_path, "img"))
+
+        instances: List[Instance] = []
+        for split in ALL_SPLITS:
+            for row in tqdm(
+                load_dataset(
+                    "neuralcatcher/hateful_memes",
+                    split="validation" if split == VALID_SPLIT else split,
+                    cache_dir=output_path,
+                )
+            ):
+                # Download the meme
+                image_path: str = row["img"]
+                local_image_path: str = os.path.join(output_path, image_path)
+                ensure_file_downloaded(
+                    source_url=self.IMAGE_URL.format(image_path=image_path),
+                    target_path=local_image_path,
+                    unpack=False,
+                )
+                # Some examples are missing images. Skip those for now
+                if not os.path.exists(local_image_path) or os.path.getsize(local_image_path) == 0:
+                    continue
+
+                content: List[MediaObject] = [
+                    MediaObject(location=local_image_path, content_type="image/jpeg"),
+                    MediaObject(text=self.QUESTION, content_type="text/plain"),
+                ]
+                answer: str = "Yes" if row["label"] == 1 else "No"
+                instances.append(
+                    Instance(
+                        Input(multimedia_content=MultimediaObject(content)),
+                        references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                        split=split,
+                    )
+                )
+
+        return instances

helm/benchmark/scenarios/vision_language/heim_human_eval_scenario.py

@@ -0,0 +1,113 @@
+from collections import Counter
+from dataclasses import dataclass
+from typing import Dict, List, Set
+import json
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    ALL_SPLITS,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+
+
+@dataclass(frozen=True)
+class HEIMHumanEvalReference(Reference):
+    # The number of human annotators who gave this reference or answer.
+    num_human_answered: int = 0
+
+
+class HEIMHumanEvalScenario(Scenario):
+    """
+    In [Holistic Evaluation of Text-To-Image Models (HEIM)](https://arxiv.org/abs/2311.04287), we evaluated 26
+    state-of-the-art text-to-image models using across 12 different aspects (e.g., toxicity mitigation, unbiasedness,
+    originality, etc.). We used human annotators through AWS Mechanical Turk to evaluate the models for some of
+    these aspects (see image below).
+    This scenario contains the AI-generated images and human annotations for the following question types:
+
+    1. Alignment
+    2. Aesthetics
+    3. Clear subject
+    4. Originality
+    5. Photorealism
+
+    Citations:
+    - HEIM: https://arxiv.org/abs/2311.04287
+    - MS COCO: https://arxiv.org/abs/1405.0312
+    """
+
+    DATASET_DOWNLOAD_URL: str = (
+        "https://worksheets.codalab.org/rest/bundles/0x502d646c366c4f1d8c4a2ccf163b958f/contents/blob/"
+    )
+    VALID_QUESTION_TYPES: Set[str] = {"alignment", "aesthetics", "clear_subject", "originality", "photorealism"}
+
+    name = "heim_human_eval"
+    description = (
+        "Images generated by text-to-image models and human annotations for HEIM "
+        "([paper](https://arxiv.org/abs/2311.04287))."
+    )
+    tags = ["vision-language", "visual question answering", "image evaluation"]
+
+    def __init__(self, question_type: str):
+        super().__init__()
+        assert question_type in self.VALID_QUESTION_TYPES, f"Invalid question type: {question_type}"
+        self._question_type: str = question_type
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the dataset
+        output_path = os.path.join(output_path, "dataset")
+        ensure_file_downloaded(
+            source_url=self.DATASET_DOWNLOAD_URL, target_path=output_path, unpack=True, unpack_type="untar"
+        )
+
+        # Load the multiple-choice questions
+        with open(os.path.join(output_path, "questions.json")) as questions_file:
+            question_info: Dict = json.load(questions_file)[self._question_type]
+
+        instances: List[Instance] = []
+        for split in ALL_SPLITS:
+            annotations_split_path: str = os.path.join(output_path, f"{self._question_type}_{split}.jsonl")
+            with open(annotations_split_path) as f:
+                # Read each line/example as a JSON object
+                for line in f.readlines():
+                    image_annotation: Dict = json.loads(line)
+                    image_path: str = os.path.join(output_path, image_annotation["image_path"])
+                    assert os.path.exists(image_path), f"Image {image_path} does not exist"
+
+                    # Get the most common human answer(s) for the question
+                    human_answers: List[str] = [str(answer) for answer in image_annotation["human_annotations"]]
+                    human_answers_to_counts = Counter(human_answers)
+                    max_count: int = max(human_answers_to_counts.values())
+                    modes: List[str] = [value for value, count in human_answers_to_counts.items() if count == max_count]
+
+                    content: List[MediaObject] = [MediaObject(location=image_path, content_type="image/png")]
+                    if "prompt" in image_annotation:
+                        # Include the prompt in the content if it exists
+                        prompt: str = image_annotation["prompt"]
+                        content.append(MediaObject(text=f"Description: {prompt}", content_type="text/plain"))
+                    content.append(MediaObject(text=question_info["question"], content_type="text/plain"))
+
+                    references: List[Reference] = [
+                        HEIMHumanEvalReference(
+                            Output(text=answer),
+                            # The mode is the most common human answer and the reference we mark as correct
+                            tags=[CORRECT_TAG] if value in modes else [],
+                            num_human_answered=human_answers_to_counts[value],
+                        )
+                        for value, answer in question_info["choices"].items()
+                    ]
+                    instances.append(
+                        Instance(
+                            Input(multimedia_content=MultimediaObject(content)),
+                            references=references,
+                            split=split,
+                        )
+                    )
+
+        return instances

helm/benchmark/scenarios/vision_language/image2structure/chart2csv_scenario.py

@@ -0,0 +1,55 @@
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+
+
+class Chart2CSVScenario(Scenario):
+    """
+    Chart to CSV
+    """
+
+    name = "chart2csv"
+    description = "Convert a chart to CSV."
+    tags = ["vision-language", "image2structure"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        assert os.path.exists(output_path), f"Dataset does not exist at {output_path}"
+        instances: List[Instance] = []
+
+        charts_path: str = os.path.join(output_path, "charts")
+        ground_truths_path: str = os.path.join(output_path, "groundtruth")
+
+        for chart_file in os.listdir(charts_path):
+            if not chart_file.endswith(".png"):
+                continue
+
+            chart_path: str = os.path.join(charts_path, chart_file)
+            ground_truth_file: str = chart_file.replace(".png", ".psv")
+            ground_truth_path: str = os.path.join(ground_truths_path, ground_truth_file)
+            assert os.path.exists(ground_truth_path), f"Ground truth does not exist at {ground_truth_path}"
+
+            content: List[MediaObject] = [
+                MediaObject(location=chart_path, content_type="image/png"),
+            ]
+            with open(ground_truth_path, "r") as file:
+                ground_truth: str = file.read().replace("|", ",")
+
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=ground_truth), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/image2structure/image2structure_scenario.py

@@ -0,0 +1,214 @@
+import os.path
+from typing import List, Optional, Dict, Any
+from abc import abstractmethod
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    ASSET_NAME_TAG,
+    ASSET_PATH_TAG,
+    TEST_SPLIT,
+    VALID_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+from helm.common.hierarchical_logger import hlog
+
+PROCESSED: str = "processed"
+
+
+class Image2StructureScenario(Scenario):
+    BASE_PROMPT: str
+    HUGGINGFACE_DATASET_NAME: str
+    SUBSETS: List[str]
+
+    name: str
+    description: str
+    tags = ["vision-language"]
+
+    helm_split_to_huggingface_split = {
+        TEST_SPLIT: "test",
+        VALID_SPLIT: "validation",
+    }
+
+    def __init__(self, subset: str, recompile_prompt: bool = True, split: str = VALID_SPLIT):
+        super().__init__()
+        assert subset in self.SUBSETS, f"Invalid subset: {subset}"
+        self._subset: str = subset
+        self._recompile_prompt: bool = recompile_prompt
+        self._split: str = split
+        self._output_path: Optional[str] = None
+
+    def preprocess_row(self, row: Dict[str, Any], assets_path: str) -> Dict[str, Any]:
+        # By default, there are no assets
+        del row["assets"]
+        row["assets_paths"] = []
+        row["assets_names"] = []
+        return row
+
+    def build_prompt(self, row: Dict[str, Any]) -> str:
+        return self.BASE_PROMPT
+
+    @abstractmethod
+    def compile_and_save(self, structure: str, assets_path: str, destination_path: str) -> str:
+        """Compile the prompt, should save the image and return the text extracted from the image"""
+        pass
+
+    def finalize(self, row: Dict[str, Any]) -> None:
+        """Perform cleanup operations after the instance has been generated."""
+        pass
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Get the instances for the scenario. This compile_and_save method should be implemented by the subclass.
+        Additionally, the subclass should implement the preprocess_row method if any preprocessing is needed.
+
+        For each instance, the following steps are performed:
+        1. Preprocess the row
+        2. Save the image locally
+            - 2.a. If we don't want to recompile the prompt, save the image directly
+            - 2.b. If we want to recompile the prompt, compile the structure and save the image
+        3. Create the prompt
+        4. Create the multimedia content
+        5. Create the reference
+        6. Finalize the Instance
+
+        Args:
+            output_path (str): The path where the instances will be saved
+
+        Returns:
+            List[Instance]: The list of instances
+        """
+        self._output_path = output_path
+        images_path: str = os.path.join(output_path, "data/images", self._subset)
+        assets_path: str = os.path.join(output_path, "data/assets", self._subset)
+        ensure_directory_exists(images_path)
+        ensure_directory_exists(assets_path)
+
+        instances: List[Instance] = []
+
+        # Process the desired set of instances
+        for row in tqdm(
+            load_dataset(
+                self.HUGGINGFACE_DATASET_NAME,
+                self._subset,
+                split=self.helm_split_to_huggingface_split[self._split],
+                cache_dir=output_path,
+            )
+        ):
+            question_uuid: str = str(row["uuid"]).replace('"', "")
+            if row["category"][1:-1] != self._subset:
+                hlog(
+                    f"Skipping instance {question_uuid} as it belong in category"
+                    f" {row['category']} and not {self._subset}"
+                )
+                continue
+
+            # Step 1: Preprocess the row
+            row = self.preprocess_row(row, assets_path)
+
+            # Step 2: Save the image locally
+            image_path: str = os.path.join(images_path, f"{question_uuid}.png")
+            if not os.path.exists(image_path):
+                if not self._recompile_prompt:  # 2.a
+                    row["image"].save(image_path)
+                else:  # 2.b
+                    if "structure" not in row:
+                        raise ValueError("Cannot recompile prompt without structure")
+                    structure: str = row["structure"]
+                    text: str = self.compile_and_save(structure, assets_path, image_path)
+                    row["text"] = text
+
+            # Step 3: Create the prompt
+            prompt: str = self.build_prompt(row)
+
+            # Step 4: Create the multimedia content
+            image_object = MediaObject(location=image_path, content_type="image/png")
+            content: List[MediaObject] = [
+                MediaObject(text=prompt, content_type="text/plain"),
+                image_object,
+            ]
+
+            # Step 5: Create the references
+            # 5.a Create the reference containing the structure and the associated image.
+            reference: Reference
+            if "structure" in row:
+                multimedia_object: MultimediaObject
+                if os.path.exists(row["structure"]):
+                    # 5.a.1 The structure is a path, therefore represent it as a multimedia object
+                    # containing the files used to compile the structure (such as a repository
+                    # containing the HTML, CSS, and JavaScript files used to generate a webpage)
+                    multimedia_object = MultimediaObject(
+                        [image_object, MediaObject(location=row["structure"], content_type="path/path")]
+                    )
+                elif row["structure"] == PROCESSED:
+                    # 5.a.2 The structure has been processed and is no longer present in the row
+                    # This can be the case if the structure is a base64 encoding of an archive that
+                    # has been extracted to a temporary path and processed but the path is no longer
+                    # existing (deleted after the processing is done)
+                    multimedia_object = MultimediaObject([image_object])
+                else:
+                    # 5.a.3 The structure is not a path, therefore it is directly a valid string
+                    # representing the structure (such as LaTeX code)
+                    multimedia_object = MultimediaObject([image_object])
+                reference = Reference(
+                    output=Output(text=row["text"], multimedia_content=multimedia_object),
+                    tags=[CORRECT_TAG],
+                )
+            else:
+                if "text" in row:
+                    reference = Reference(
+                        output=Output(text=row["text"], multimedia_content=MultimediaObject([image_object])),
+                        tags=[CORRECT_TAG],
+                    )
+                else:
+                    reference = Reference(
+                        output=Output(multimedia_content=MultimediaObject([image_object])), tags=[CORRECT_TAG]
+                    )
+            references: List[Reference] = [reference]
+
+            # 5.b Create the reference containing the assets
+            if len(row["assets_paths"]) > 0:
+                assets_paths_reference: Reference = Reference(
+                    output=Output(
+                        text=", ".join(
+                            row["assets_paths"]
+                        ),  # TODO: This is for debugging purposes (to show in the frontend)
+                        multimedia_content=MultimediaObject(
+                            [
+                                MediaObject(location=asset, content_type=f"image/{asset.split('.')[-1].lower()}")
+                                for asset in row["assets_paths"]
+                            ]
+                        ),
+                    ),
+                    tags=[ASSET_PATH_TAG],
+                )
+                references.append(assets_paths_reference)
+                assets_names_reference: Reference = Reference(
+                    output=Output(
+                        text=", ".join(
+                            row["assets_names"]
+                        ),  # TODO: This is for debugging purposes (to show in the frontend)
+                        multimedia_content=MultimediaObject(
+                            [MediaObject(text=asset, content_type="text/plain") for asset in row["assets_names"]]
+                        ),
+                    ),
+                    tags=[ASSET_NAME_TAG],
+                )
+                references.append(assets_names_reference)
+
+            # Step 6: Finalize the Instance
+            self.finalize(row)
+            instance = Instance(
+                input=Input(multimedia_content=MultimediaObject(content)), references=references, split=self._split
+            )
+            instances.append(instance)
+
+        assert len(instances) > 0, f"No instances found for subject {self._subset}"
+        return instances

helm/benchmark/scenarios/vision_language/image2structure/latex_scenario.py

@@ -0,0 +1,25 @@
+from helm.benchmark.scenarios.scenario import VALID_SPLIT
+from helm.benchmark.scenarios.vision_language.image2structure.utils_latex import (
+    latex_to_image,
+    strip_unnecessary_latex_parts,
+)
+from helm.benchmark.scenarios.vision_language.image2structure.image2structure_scenario import Image2StructureScenario
+
+
+class LatexScenario(Image2StructureScenario):
+    BASE_PROMPT = "Please provide the LaTeX code used to generate this image. Only generate the code relevant to what you see. Your code will be surrounded by all the imports necessary as well as the begin and end document delimiters."  # noqa: E501
+    HUGGINGFACE_DATASET_NAME = "stanford-crfm/i2s-latex"
+    SUBSETS = ["equation", "table", "plot", "algorithm"]
+
+    name = "image2latex"
+    description = "Evaluate multimodal models on Latex generation to recreate a provided image"
+
+    def __init__(self, subset: str, recompile_prompt: bool = True, split: str = VALID_SPLIT):
+        super().__init__(subset, recompile_prompt, split)
+
+    def compile_and_save(self, structure: str, assets_path: str, destination_path: str) -> str:
+        image, infos = latex_to_image(structure, assets_path=assets_path, crop=True)
+        image.save(destination_path)
+        assert "latex_code" in infos
+        text: str = strip_unnecessary_latex_parts(infos["latex_code"])
+        return text

helm/benchmark/scenarios/vision_language/image2structure/musicsheet_scenario.py

@@ -0,0 +1,20 @@
+from helm.benchmark.scenarios.scenario import VALID_SPLIT
+from helm.benchmark.scenarios.vision_language.image2structure.image2structure_scenario import Image2StructureScenario
+
+
+class MusicSheetScenario(Image2StructureScenario):
+    BASE_PROMPT = (
+        "Please generate the Lilypond code to generate a music sheet that looks like this image as much as feasible possible.\n"  # noqa: E501
+        "This music sheet was created by me, and I would like to recreate it using Lilypond."
+    )
+    HUGGINGFACE_DATASET_NAME = "stanford-crfm/i2s-musicsheet"
+    SUBSETS = ["music"]
+
+    name = "image2musicsheet"
+    description = "Evaluate multimodal models on Lilypond generation to recreate a provided image"
+
+    def __init__(self, subset: str, recompile_prompt: bool = True, split: str = VALID_SPLIT):
+        super().__init__(subset, recompile_prompt, split)
+
+    def compile_and_save(self, structure: str, assets_path: str, destination_path: str) -> str:
+        raise Exception("Music sheets have no ground truth, compilation is not possible")