crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/benchmark/scenarios/vision_language/mme_scenario.py

@@ -0,0 +1,145 @@
+import os.path
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+
+
+class MMEScenario(Scenario):
+    """
+        MME: A Comprehensive Evaluation Benchmark for Multimodal Large Language Models
+
+        Multimodal Large Language Model (MLLM) relies on the powerful LLM to perform
+        multimodal tasks, showing amazing emergent abilities in recent studies. However,
+        it is difficult for these case studies to fully reflect the performance of MLLM,
+        lacking a comprehensive evaluation. In MME, we fill in this blank, presenting
+        the first comprehensive MLLM Evaluation benchmark MME. It measures both perception
+        and cognition abilities on a total of 14 subtasks. In order to avoid data leakage
+        that may arise from direct use of public datasets for evaluation, the annotations
+        of instruction-answer pairs are all manually designed. The concise instruction design
+        allows us to fairly compare MLLMs, instead of struggling in prompt engineering.
+        Besides, with such an instruction, we can also easily carry out quantitative
+        statistics. We rephrase the answer type of MME to multiple-choice question-answering.
+        We use the multiple-choice metrics for 14 different evaluation tasks.
+
+        @article{fu2023mme,
+        title={MME: A Comprehensive Evaluation Benchmark for Multimodal Large Language Models},
+        author={Fu, Chaoyou and Chen, Peixian and Shen, Yunhang and Qin, Yulei and
+        Zhang, Mengdan and Lin, Xu and Yang, Jinrui and Zheng, Xiawu and Li, Ke and
+        Sun, Xing and Wu, Yunsheng and Ji, Rongrong},
+        journal={arXiv preprint arXiv:2306.13394},
+        year={2023}
+    }
+
+        Paper: https://arxiv.org/abs/2306.13394
+    """
+
+    MME_HUGGINGFACE_DATASET_NAME: str = "lmms-lab/MME"
+
+    SUBJECTS: List[str] = [
+        "existence",
+        "scene",
+        "posters",
+        "color",
+        "OCR",
+        "position",
+        "celebrity",
+        "artwork",
+        "commonsense_reasoning",
+        "numerical_calculation",
+        "landmark",
+        "count",
+        "text_translation",
+        "code_reasoning",
+    ]
+
+    name = "mme"
+    description = "Evaluate multimodal models on  ([paper](https://arxiv.org/abs/2306.13394))."
+    tags = ["vision-language"]
+    options: List[str] = ["Yes", "No"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+    def get_label_from_answer(self, answer: str):
+        label: str
+        if answer == "Yes":
+            label = "A"
+        elif answer == "No":
+            label = "B"
+        else:
+            raise NotImplementedError(f"Invalid answer: {answer}")
+        return label
+
+    def remove_question_suffix_for_mcqa(self, question: str):
+        return question.replace("Please answer yes or no.", "").strip()
+
+    def get_question_id(self, question_id: str):
+        return question_id.split(".")[0].replace("/", "-")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        # There is only the test split in Unicorn benchmark
+        instances: List[Instance] = []
+        # Process the test set
+        # Two open-ended generation instances and
+        # one multi-choice generation instance per row
+        for row in tqdm(
+            load_dataset(
+                self.MME_HUGGINGFACE_DATASET_NAME,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            if row["category"] != self._subject:
+                continue
+            question_id: str = self.get_question_id(row["question_id"])
+            # Save the image locally
+            image_path: str = os.path.join(images_path, f"{question_id}.png")
+            if not os.path.exists(image_path):
+                row["image"].save(image_path)
+
+            question: str = self.remove_question_suffix_for_mcqa(row["question"])
+            answer: str = row["answer"]
+            references: List[Reference] = []
+
+            answer = self.get_label_from_answer(answer)
+            # The given correct answer is a letter, but we need an index
+            correct_answer_index: int = ord(answer) - ord("A")
+            # The options are originally appended to the question
+
+            for i, option in enumerate(self.options):
+                reference: Reference
+                is_correct: bool = i == correct_answer_index
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if is_correct else [])
+                references.append(reference)
+
+            content = [
+                MediaObject(location=image_path, content_type="image/png"),
+                MediaObject(text=question, content_type="text/plain"),
+            ]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=references,
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/mmmu_scenario.py

@@ -0,0 +1,187 @@
+import os.path
+from typing import Dict, List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+
+
+class MMMUScenario(Scenario):
+    """
+    MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI
+
+    We introduce MMMU: a new benchmark designed to evaluate multimodal models on massive multi-discipline
+    tasks demanding college-level subject knowledge and deliberate reasoning. MMMU includes 11.5K meticulously
+    collected multimodal questions from college exams, quizzes, and textbooks, covering six core disciplines:
+    Art & Design, Business, Science, Health & Medicine, Humanities & Social Science, and Tech & Engineering.
+    These questions span 30 subjects and 183 subfields, comprising 30 highly heterogeneous image types, such
+    as charts, diagrams, maps, tables, music sheets, and chemical structures.
+
+      @article{yue2023mmmu,
+        title={MMMU: A Massive Multi-discipline Multimodal Understanding and Reasoning Benchmark for Expert AGI},
+        author={Xiang Yue and Yuansheng Ni and Kai Zhang and Tianyu Zheng and Ruoqi Liu and Ge Zhang and Samuel
+        Stevens and Dongfu Jiang and Weiming Ren and Yuxuan Sun and Cong Wei and Botao Yu and Ruibin Yuan and
+        Renliang Sun and Ming Yin and Boyuan Zheng and Zhenzhu Yang and Yibo Liu and Wenhao Huang and Huan Sun
+        and Yu Su and Wenhu Chen},
+        journal={arXiv preprint arXiv:2311.16502},
+        year={2023},
+      }
+
+    Paper: https://arxiv.org/abs/2311.16502
+    Website: https://mmmu-benchmark.github.io/
+    """
+
+    MMMU_HUGGINGFACE_DATASET_NAME: str = "MMMU/MMMU"
+    MAX_NUM_IMAGES: int = 7
+
+    SUBJECTS: List[str] = [
+        "Accounting",
+        "Agriculture",
+        "Architecture_and_Engineering",
+        "Art",
+        "Art_Theory",
+        "Basic_Medical_Science",
+        "Biology",
+        "Chemistry",
+        "Clinical_Medicine",
+        "Computer_Science",
+        "Design",
+        "Diagnostics_and_Laboratory_Medicine",
+        "Economics",
+        "Electronics",
+        "Energy_and_Power",
+        "Finance",
+        "Geography",
+        "History",
+        "Literature",
+        "Manage",
+        "Marketing",
+        "Materials",
+        "Math",
+        "Mechanical_Engineering",
+        "Music",
+        "Pharmacy",
+        "Physics",
+        "Psychology",
+        "Public_Health",
+        "Sociology",
+    ]
+
+    name = "mmmu"
+    description = (
+        "Evaluate multimodal models on massive multi-discipline tasks demanding college-level "
+        "subject knowledge and deliberate reasoning ([paper](https://arxiv.org/abs/2311.16502))."
+    )
+    tags = ["vision-language"]
+
+    def __init__(self, subject: str, question_type: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+        assert question_type in ["multiple-choice", "open"], f"Invalid question type: {question_type}"
+        self._question_type: str = question_type
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images", self._subject)
+        ensure_directory_exists(images_path)
+
+        instances: List[Instance] = []
+
+        # Process the validation set
+        # There seems to be a dev set, but it's unavailable through load_dataset.
+        # When loading "dev", we get error: ValueError: Unknown split "dev". Should be one of ['test', 'validation'].
+        # The test set doesn't have answers, since the MMMU competition/leaderboard uses the test set
+        for row in tqdm(
+            load_dataset(self.MMMU_HUGGINGFACE_DATASET_NAME, self._subject, split="validation", cache_dir=output_path)
+        ):
+            # Skip questions that aren't in the subject we're interested in
+            if row["question_type"] != self._question_type:
+                continue
+
+            question_id: str = row["id"]
+            if self._subject not in question_id:
+                continue
+
+            question_template: str = row["question"]
+            options: List[str] = eval(row["options"])  # Convert the string to list of options
+            answer: str = row["answer"]
+
+            # Create the question. Questions can have text and images interleaved
+            question_template_to_image_path: Dict[str, str] = {}
+            content: List[MediaObject] = []
+            for img_number in range(1, self.MAX_NUM_IMAGES):
+                image_id: str = f"image_{img_number}"
+                if row[image_id] is None:
+                    # At this point, there are no more images for this question
+                    break
+
+                # Save the image locally
+                image_path: str = os.path.join(images_path, f"{question_id}_{image_id}.png")
+                if not os.path.exists(image_path):
+                    row[image_id].save(image_path)
+
+                image_template_tag: str = f"<image {img_number}>"
+                question_template_to_image_path[image_template_tag] = image_path
+
+                # There are cases when the image is included, but it is not used either in the
+                # question template or in the answer options
+                if image_template_tag not in question_template:
+                    # The image is not in the question template
+                    continue
+
+                head, question_template = question_template.split(image_template_tag, 1)
+                if head:
+                    content.append(MediaObject(text=head, content_type="text/plain"))
+                content.append(MediaObject(location=image_path, content_type="image/png"))
+
+            # Add the rest of the question template
+            if question_template:
+                content.append(MediaObject(text=question_template, content_type="text/plain"))
+
+            # Add the references
+            references: List[Reference] = []
+            if self._question_type == "multiple-choice":
+                # The given correct answer is a letter, but we need an index
+                correct_answer_index: int = ord(answer) - ord("A")
+
+                for i, option in enumerate(options):
+                    reference: Reference
+                    is_correct: bool = i == correct_answer_index
+
+                    if option in question_template_to_image_path:
+                        # Image answer choices
+                        image_path = question_template_to_image_path[option]
+                        image_choice = MediaObject(location=image_path, content_type="image/png")
+                        reference = Reference(
+                            Output(multimedia_content=MultimediaObject([image_choice])),
+                            tags=[CORRECT_TAG] if is_correct else [],
+                        )
+                    else:
+                        # Text answer choices
+                        reference = Reference(Output(text=option), tags=[CORRECT_TAG] if is_correct else [])
+
+                    references.append(reference)
+            else:
+                references.append(Reference(Output(text=answer), tags=[CORRECT_TAG]))
+
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=references,
+                    split=VALID_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/mscoco_captioning_scenario.py

@@ -0,0 +1,92 @@
+import json
+import os
+from collections import defaultdict
+from typing import Any, Dict, List
+
+from helm.common.general import ensure_file_downloaded
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+)
+
+
+class MSCOCOCaptioningScenario(Scenario):
+    """
+    Microsoft COCO (MS-COCO) is a large-scale object detection, segmentation, and captioning dataset.
+    It has 330K images, with over 200K of them labeled. We use the 2014 version of the dataset instead
+    of the 2017 version because of the larger validation set. According to https://cocodataset.org/#download,
+    the 2014 version has 83K images in the train split and 41K in the val split.
+
+    Each image also has five captions. For example, image #335111 has the following five captions:
+        1. a row of bikes on the sidewalk, 2 on the ground.
+        2. a couple of bikes laying on their sides on a sidewalk.
+        3. a person wearing a black coat with a hood stands on the street, near many bikes
+        4. a woman standing in front of a row of bicycles in front of a bus stop with two bikes knocked over
+        5. there are some bicycles laying on their sides
+
+    Paper: https://arxiv.org/abs/1405.0312
+    Website: https://cocodataset.org/#home
+    """
+
+    ANNOTATIONS_DOWNLOAD_URL: str = "http://images.cocodataset.org/annotations/annotations_trainval2014.zip"
+    SPLIT_DOWNLOAD_URL_TEMPLATE: str = "http://images.cocodataset.org/zips/{split}2014.zip"
+    COCO_SPLIT_TO_HELM_SPLIT: Dict[str, str] = {"train": TRAIN_SPLIT, "val": VALID_SPLIT}
+
+    name = "mscoco"
+    description = "Microsoft COCO: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))."
+    tags = ["text-to-image", "image-to-text"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the annotations which contains the image IDs, filenames and captions
+        data_path: str = os.path.join(output_path, "data")
+        ensure_file_downloaded(source_url=self.ANNOTATIONS_DOWNLOAD_URL, target_path=data_path, unpack=True)
+
+        instances: List[Instance] = []
+        for coco_split, helm_split in self.COCO_SPLIT_TO_HELM_SPLIT.items():
+            # Download the images of the split
+            split_url: str = self.SPLIT_DOWNLOAD_URL_TEMPLATE.format(split=coco_split)
+            split_path: str = os.path.join(data_path, coco_split)
+            ensure_file_downloaded(source_url=split_url, target_path=split_path, unpack=True)
+
+            # Read the metadata for the split
+            metadata_path: str = os.path.join(data_path, f"captions_{coco_split}2014.json")
+            with open(metadata_path, "r") as f:
+                metadata: Dict[str, Any] = json.load(f)
+
+            # Get the path of each image
+            image_id_to_path: Dict[int, str] = {
+                image_metadata["id"]: os.path.join(split_path, image_metadata["file_name"])
+                for image_metadata in metadata["images"]
+            }
+
+            # Gather the five captions for each image
+            image_id_to_captions: Dict[int, List[str]] = defaultdict(list)
+            for annotation in metadata["annotations"]:
+                image_id_to_captions[annotation["image_id"]].append(annotation["caption"])
+
+            # Create instances
+            for image_id in image_id_to_path:
+                image_path: str = image_id_to_path[image_id]
+                captions: List[str] = image_id_to_captions[image_id]
+
+                content: List[MediaObject] = [
+                    MediaObject(location=image_path, content_type="image/jpeg"),
+                ]
+                instances.append(
+                    Instance(
+                        Input(multimedia_content=MultimediaObject(content)),
+                        references=[
+                            Reference(Output(text=caption.rstrip()), tags=[CORRECT_TAG]) for caption in captions
+                        ],
+                        split=helm_split,
+                    )
+                )
+
+        return instances

helm/benchmark/scenarios/vision_language/mscoco_categorization_scenario.py

@@ -0,0 +1,117 @@
+import json
+import os
+from collections import defaultdict
+from typing import Any, Dict, List, Set
+
+from helm.common.general import ensure_file_downloaded
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+)
+
+
+class MSCOCOCategorizationScenario(Scenario):
+    """
+    Microsoft COCO (MS-COCO) is a large-scale object detection, segmentation, and captioning dataset.
+    It has 330K images, with over 200K of them labeled. We use the 2017 version of the dataset
+    for the categorization task.
+
+    Paper: https://arxiv.org/abs/1405.0312
+    Website: https://cocodataset.org/#home
+    """
+
+    ANNOTATIONS_DOWNLOAD_URL: str = "http://images.cocodataset.org/annotations/stuff_annotations_trainval2017.zip"
+    SPLIT_DOWNLOAD_URL_TEMPLATE: str = "http://images.cocodataset.org/zips/{split}2017.zip"
+    COCO_SPLIT_TO_HELM_SPLIT: Dict[str, str] = {"train": TRAIN_SPLIT, "val": VALID_SPLIT}
+
+    name = "mscoco"
+    description = "Microsoft COCO: Common Objects in Context ([paper](https://arxiv.org/abs/1405.0312))."
+    tags = ["text-to-image", "image-to-text"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the annotations which contains the image IDs, filenames and captions
+        data_path: str = os.path.join(output_path, "data_2017")
+        ensure_file_downloaded(source_url=self.ANNOTATIONS_DOWNLOAD_URL, target_path=data_path, unpack=True)
+
+        super_categories_to_categories: Dict[str, List[str]] = defaultdict(list)
+        category_id_to_category: Dict[int, str] = {}
+        category_id_to_super_category: Dict[int, str] = {}
+
+        instances: List[Instance] = []
+        for coco_split, helm_split in self.COCO_SPLIT_TO_HELM_SPLIT.items():
+            # Download the images of the split
+            split_url: str = self.SPLIT_DOWNLOAD_URL_TEMPLATE.format(split=coco_split)
+            split_path: str = os.path.join(data_path, coco_split)
+            ensure_file_downloaded(source_url=split_url, target_path=split_path, unpack=True)
+
+            # Read the metadata for the split
+            metadata_path: str = os.path.join(data_path, f"stuff_{coco_split}2017.json")
+            with open(metadata_path, "r") as f:
+                metadata: Dict[str, Any] = json.load(f)
+
+            for category_metadata in metadata["categories"]:
+                # Each metadata looks like this {'supercategory': 'textile', 'id': 92, 'name': 'banner'}
+                category_id: int = category_metadata["id"]
+                category: str = category_metadata["name"]
+                super_category: str = category_metadata["supercategory"]
+                super_categories_to_categories[super_category].append(category)
+                category_id_to_category[category_id] = category
+                category_id_to_super_category[category_id] = super_category
+
+            # Get the path of each image
+            image_id_to_path: Dict[int, str] = {
+                image_metadata["id"]: os.path.join(split_path, image_metadata["file_name"])
+                for image_metadata in metadata["images"]
+            }
+
+            # Gather the five captions for each image
+            image_id_to_category_ids: Dict[int, List[int]] = defaultdict(list)
+            for annotation in metadata["annotations"]:
+                image_id_to_category_ids[annotation["image_id"]].append(annotation["category_id"])
+
+            # Create instances
+            for image_id in image_id_to_path:
+                image_path: str = image_id_to_path[image_id]
+                assert os.path.exists(image_path), f"Image path {image_path} does not exist"
+                category_ids: List[int] = image_id_to_category_ids[image_id]
+
+                content: List[MediaObject] = [
+                    MediaObject(location=image_path, content_type="image/jpeg"),
+                ]
+                references: List[Reference] = []
+                correct_super_categories: Set[str] = set(
+                    category_id_to_super_category[category_id] for category_id in category_ids
+                )
+                # for category_id in category_ids:
+                #     category = category_id_to_category[category_id]
+                #     super_category = category_id_to_super_category[category_id]
+                #     references.extend(
+                #         [
+                #             Reference(Output(text=category), tags=[CORRECT_TAG]),
+                #             Reference(Output(text=super_category), tags=[CORRECT_TAG]),
+                #         ]
+                #     )
+                for super_category in super_categories_to_categories:
+                    references.append(
+                        Reference(
+                            Output(text=super_category),
+                            tags=[CORRECT_TAG] if super_category in correct_super_categories else [],
+                        )
+                    )
+
+                instances.append(
+                    Instance(
+                        Input(multimedia_content=MultimediaObject(content)),
+                        references=references,
+                        split=helm_split,
+                    )
+                )
+
+        return instances