crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/scenarios/vision_language/seed_bench_scenario.py

@@ -0,0 +1,129 @@
+import os.path
+from typing import Dict, List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+
+
+class SEEDBenchScenario(Scenario):
+    """
+    SEED-Bench: Benchmarking Multimodal LLMs with Generative Comprehension
+
+    Based on powerful Large Language Models (LLMs), recent generative Multimodal
+    Large Language Models (MLLMs) have gained prominence as a pivotal research area.
+    In Seed-Bench, we address the evaluation of generative comprehension in MLLMs
+    as a preliminary step towards a comprehensive assessment of generative models.
+    SEED-Bench consists of 19K multiple choice questions with accurate human annotations
+    (x 6 larger than existing benchmarks), which spans 12 evaluation dimensions
+    including the comprehension of both the image and video modality. We select 9
+    evaluation aspects that take image as the input. In the benchmark,
+    Multiple-choice questions with groundtruth options derived from human
+    annotation enables an objective and efficient assessment of model performance,
+    eliminating the need for human or GPT intervention during evaluation. We employ
+    the multiple-choice metric for evaluating the performance of models.
+
+    @article{li2023seed,
+    title={Seed-bench: Benchmarking multimodal llms with generative comprehension},
+    author={Li, Bohao and Wang, Rui and Wang, Guangzhi and Ge, Yuying and Ge, Yixiao and Shan, Ying},
+    journal={arXiv preprint arXiv:2307.16125},
+    year={2023}
+    }
+
+    Paper: https://arxiv.org/abs/2307.16125
+    """
+
+    SEED_BENCH_HUGGINGFACE_DATASET_NAME: str = "lmms-lab/SEED-Bench"
+
+    SUBJECTS: Dict[str, int] = {
+        "scene-understanding": 1,
+        "instance-identity": 2,
+        "instance-attributes": 3,
+        "instance-location": 4,
+        "instances-counting": 5,
+        "spatial-relation": 6,
+        "instance-interaction": 7,
+        "visual-reasoning": 8,
+        "text-understanding": 9,
+    }
+
+    name = "seed_bench"
+    description = "Evaluate multimodal models on  ([paper](https://arxiv.org/abs/2307.16125))."
+    tags = ["vision-language"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+    def get_subject_name(self, subject_name: str) -> str:
+        return "-".join(subject_name.lower().split())
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        # There is only the test split in Unicorn benchmark
+        instances: List[Instance] = []
+        # Process the test set
+        # Two open-ended generation instances and
+        # one multi-choice generation instance per row
+        for row in tqdm(
+            load_dataset(
+                self.SEED_BENCH_HUGGINGFACE_DATASET_NAME,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            question_type_key: str = self.get_subject_name(self._subject)
+            if row["question_type_id"] != self.SUBJECTS[question_type_key]:
+                continue
+            question_id: str = row["question_id"]
+            # Download the image
+            # Save the image locally
+            image_path: str = os.path.join(images_path, f"{question_id}.png")
+            if not os.path.exists(image_path):
+                # some images are CMYK mode, convert to RGB.
+                row["image"][0].convert("RGB").save(image_path, "PNG", optimize=True)
+
+            # Add the references
+            references: List[Reference] = []
+            question: str = row["question"]
+            answer: str
+            content: List[MediaObject]
+            options: List[str] = [row["choice_a"], row["choice_b"], row["choice_c"], row["choice_d"]]
+            answer = row["answer"].strip()
+            # The given correct answer is a letter, but we need an index
+            correct_answer_index: int = ord(answer) - ord("A")
+            # The options are originally appended to the question
+
+            for i, option in enumerate(options):
+                reference: Reference
+                is_correct: bool = i == correct_answer_index
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if is_correct else [])
+                references.append(reference)
+
+            content = [
+                MediaObject(location=image_path, content_type="image/png"),
+                MediaObject(text=question, content_type="text/plain"),
+            ]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=references,
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/unicorn_scenario.py

@@ -0,0 +1,108 @@
+import os.path
+from typing import Dict, List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class UnicornScenario(Scenario):
+    """
+    How Many Unicorns are in this Image? A Safety Evaluation Benchmark of Vision LLMs
+
+    We shift our focus from evaluating standard performance to introducing a comprehensive safety evaluation
+    suite Unicorn, covering both out-of-distribution (OOD) generalization and adversarial robustness. For the OOD
+    evaluation, we present two novel VQA datasets --- OODCV-VQA and Sketchy-VQA, each with one variant, designed
+    to test model performance under challenging conditions. In the OOD scenario, questions are matched with
+    boolean or numerical answers, and we use exact match metrics for evaluation. When comparing OOD Sketchy-VQA
+    with its synthesized in-distribution counterpart, we found an average model output F1 drop of 8.9%,
+    highlighting the challenging nature of the OOD scenario in the Unicorn benchmark.
+
+    @article{tu2023unicorns,
+    title={How Many Unicorns Are in This Image? A Safety Evaluation Benchmark for Vision LLMs},
+    author={Tu, Haoqin and Cui, Chenhang and Wang, Zijun and Zhou, Yiyang and Zhao, Bingchen and Han,
+    Junlin and Zhou, Wangchunshu and Yao, Huaxiu and Xie, Cihang},
+    journal={arXiv preprint arXiv:2311.16101},
+    year={2023}
+    }
+
+    Paper: https://arxiv.org/abs/2311.16101
+    """
+
+    UNICORN_HUGGINGFACE_DATASET_NAME: str = "PahaII/unicorn"
+
+    IMAGE_URL: str = "https://huggingface.co/datasets/PahaII/unicorn/resolve/main/images/{image_path}?download=true"
+
+    SUBJECTS: List[str] = ["OODCV-VQA", "OODCV-Counterfactual", "Sketchy-VQA", "Sketchy-Challenging"]
+
+    IMG_TYPE: Dict[str, str] = {
+        "OODCV-VQA": "jpeg",
+        "OODCV-Counterfactual": "jpeg",
+        "Sketchy-VQA": "png",
+        "Sketchy-Challenging": "png",
+    }
+
+    name = "unicorn"
+    description = (
+        "Evaluate multimodal models on two out-of-distribution scenarios with four subjects"
+        " ([paper](https://arxiv.org/abs/2311.16101))."
+    )
+    tags = ["vision-language"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+        self._image_type: str = self.IMG_TYPE[self._subject]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        # There is only the test split in Unicorn benchmark
+        instances: List[Instance] = []
+        question_data_files = {TEST_SPLIT: f"{self._subject}.json"}
+
+        # Process the test set
+        for row in tqdm(
+            load_dataset(
+                self.UNICORN_HUGGINGFACE_DATASET_NAME,
+                data_files=question_data_files,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            # Download the image
+            image_path: str = row["image_path"]
+            local_image_path: str = os.path.join(output_path, image_path)
+            ensure_file_downloaded(
+                source_url=self.IMAGE_URL.format(image_path=image_path),
+                target_path=local_image_path,
+                unpack=False,
+            )
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type=f"image/{self._image_type}"),
+                MediaObject(text=row["question"], content_type="text/plain"),
+            ]
+            answer: str = row["answer"]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/viz_wiz_scenario.py

@@ -0,0 +1,107 @@
+from typing import Dict, List, Set
+import json
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+
+
+class VizWizScenario(Scenario):
+    """
+    VizWiz is a real-world visual question answering dataset consisting of questions
+    asked by people who are blind. It originates from a natural visual question answering
+    setting where blind people each took an image and recorded a spoken question about it,
+    together with 10 crowdsourced answers per visual question.
+
+    Version as of January 1, 2020:
+
+    - 20,523 training image/question pairs
+    - 205,230 training answer/answer confidence pairs
+    - 4,319 validation image/question pairs
+    - 43,190 validation answer/answer confidence pairs
+
+    where answer confidences are one of {"yes", "maybe", "no"}.
+
+    Answers are publicly shared for the train and validation splits and hidden for the test split.
+
+    Paper: https://arxiv.org/abs/1802.08218
+    Website: https://vizwiz.org/tasks-and-datasets/vqa
+    """
+
+    # Annotations are not available for the test set
+    ANNOTATIONS_URL: str = "https://vizwiz.cs.colorado.edu/VizWiz_final/vqa_data/Annotations.zip"
+    SPLIT_TO_ANNOTATIONS_FILE: Dict[str, str] = {
+        TRAIN_SPLIT: "train.json",
+        VALID_SPLIT: "val.json",
+    }
+
+    SPLIT_TO_IMAGES: Dict[str, str] = {
+        TRAIN_SPLIT: "https://vizwiz.cs.colorado.edu/VizWiz_final/images/train.zip",
+        VALID_SPLIT: "https://vizwiz.cs.colorado.edu/VizWiz_final/images/val.zip",
+    }
+
+    name = "viz_wiz"
+    description = (
+        "Real-world VQA dataset consisting of questions asked by "
+        "people who are blind ([paper](https://arxiv.org/abs/1802.08218))."
+    )
+    tags = ["vision-language", "visual question answering"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the questions and annotations
+        annotations_path: str = os.path.join(output_path, "annotations")
+        ensure_file_downloaded(
+            source_url=self.ANNOTATIONS_URL,
+            target_path=annotations_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+
+        instances: List[Instance] = []
+        for split in [TRAIN_SPLIT, VALID_SPLIT]:
+            # Download the images for the split
+            images_path: str = os.path.join(output_path, split)
+            ensure_file_downloaded(
+                source_url=self.SPLIT_TO_IMAGES[split],
+                target_path=images_path,
+                unpack=True,
+                unpack_type="unzip",
+            )
+
+            annotations_split_path: str = os.path.join(annotations_path, self.SPLIT_TO_ANNOTATIONS_FILE[split])
+            with open(annotations_split_path) as f:
+                for image_annotation in json.load(f):
+                    image_path: str = os.path.join(images_path, image_annotation["image"])
+                    assert os.path.exists(image_path), f"Image {image_path} does not exist"
+
+                    content: List[MediaObject] = [
+                        MediaObject(location=image_path, content_type="image/jpeg"),
+                        MediaObject(text=image_annotation["question"], content_type="text/plain"),
+                    ]
+                    deduped_answers: Set[str] = {
+                        answer_json["answer"]
+                        for answer_json in image_annotation["answers"]
+                        if answer_json["answer_confidence"] == "yes"
+                    }
+
+                    instances.append(
+                        Instance(
+                            Input(multimedia_content=MultimediaObject(content)),
+                            references=[
+                                Reference(Output(text=answer), tags=[CORRECT_TAG]) for answer in deduped_answers
+                            ],
+                            split=split,
+                        )
+                    )
+
+        return instances

helm/benchmark/scenarios/vision_language/vqa_scenario.py

@@ -54,7 +54,7 @@ class VQAScenario(Scenario):
         TEST_SPLIT: "http://images.cocodataset.org/zips/test2015.zip",
     }
 
-    name = "visual_question_answering"
+    name = "vqa"
     description = "Open-ended questions about images ([paper](https://arxiv.org/abs/1612.00837))."
     tags = ["vision-language", "visual question answering"]
 

helm/benchmark/scenarios/wmt_14_scenario.py

@@ -1,6 +1,6 @@
 from typing import List, Any
 from datasets import load_dataset
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import htrack_block
 from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
 
 
@@ -59,26 +59,26 @@ class WMT14Scenario(Scenario):
         return deduplicated_dataset
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        hlog("Loading the HuggingFace dataset. The first time could take several minutes.")
-        subset_name = f"{self.source_language if self.source_language!='en' else self.target_language}-en"
-        hf_dataset: Any = load_dataset("wmt14", subset_name)
-        splits = {"train": TRAIN_SPLIT, "validation": VALID_SPLIT, "test": TEST_SPLIT}
+        with htrack_block("Loading the HuggingFace dataset. The first time could take several minutes."):
+            subset_name = f"{self.source_language if self.source_language!='en' else self.target_language}-en"
+            hf_dataset: Any = load_dataset("wmt14", subset_name, trust_remote_code=True)
+            splits = {"train": TRAIN_SPLIT, "validation": VALID_SPLIT, "test": TEST_SPLIT}
 
         instances: List[Instance] = []
-        hlog("Generating instances")
-        # Some training sets are too large, so we will only take a random subset of it.
-        hf_dataset["train"] = hf_dataset["train"].shuffle(seed=42)[:MAX_TRAIN_INSTANCES]
-        hf_dataset["train"]["translation"] = self._deduplicate(hf_dataset["train"]["translation"])
-        for example in hf_dataset["train"]["translation"]:
-            source_sentence: str = example[self.source_language]
-            target_sentence: str = example[self.target_language]
-            instances.append(
-                Instance(
-                    input=Input(text=source_sentence),
-                    references=[Reference(Output(text=target_sentence), tags=[CORRECT_TAG])],
-                    split="train",
+        with htrack_block("Generating instances"):
+            # Some training sets are too large, so we will only take a random subset of it.
+            hf_dataset["train"] = hf_dataset["train"].shuffle(seed=42)[:MAX_TRAIN_INSTANCES]
+            hf_dataset["train"]["translation"] = self._deduplicate(hf_dataset["train"]["translation"])
+            for example in hf_dataset["train"]["translation"]:
+                source_sentence: str = example[self.source_language]
+                target_sentence: str = example[self.target_language]
+                instances.append(
+                    Instance(
+                        input=Input(text=source_sentence),
+                        references=[Reference(Output(text=target_sentence), tags=[CORRECT_TAG])],
+                        split="train",
+                    )
                 )
-            )
 
         # No special handling needed for validation or test.
         for split_name in ["validation", "test"]:

helm/benchmark/server.py

@@ -5,10 +5,15 @@ Starts a local HTTP server to display benchmarking assets.
 
 import argparse
 import importlib_resources as resources
+import json
 from os import path
 import urllib
 
-from bottle import Bottle, static_file
+from bottle import Bottle, static_file, HTTPResponse
+import yaml
+
+from helm.benchmark.presentation.schema import SCHEMA_CLASSIC_YAML_FILENAME
+from helm.common.general import serialize_dates
 
 
 app = Bottle()
@@ -28,6 +33,35 @@ def serve_config():
         )
 
 
+# Shim for running helm-server for old suites from old version of helm-summarize
+# that do not contain schema.json.
+#
+# The HELM web frontend expects to find a schema.json at /benchmark_output/runs/<version>/schema.json
+# which is produced by the new version of helm-summarize but not the old version.
+# When serving a suite produced by the old version of helm-summarize, the schena.json will be missing.
+# This shim supports those suites by serving a schena.json that is dynamically computed from schema_classic.yaml
+#
+# We will remove this in a few months after most users have moved to the new version of helm-summarize.
+#
+# TODO(2024-03-01): Remove this.
+@app.get("/benchmark_output/<runs_or_releases:re:runs|releases>/<version>/schema.json")
+def server_schema(runs_or_releases, version):
+    relative_schema_path = path.join(runs_or_releases, version, "schema.json")
+    absolute_schema_path = path.join(app.config["helm.outputpath"], relative_schema_path)
+    if path.isfile(absolute_schema_path):
+        response = static_file(relative_schema_path, root=app.config["helm.outputpath"])
+    else:
+        # Suite does not contain schema.json
+        # Fall back to schema_classic.yaml from the static directory
+        classic_schema_path = path.join(app.config["helm.staticpath"], SCHEMA_CLASSIC_YAML_FILENAME)
+        with open(classic_schema_path, "r") as f:
+            response = HTTPResponse(json.dumps(yaml.safe_load(f), indent=2, default=serialize_dates))
+    response.set_header("Cache-Control", "no-cache, no-store, must-revalidate")
+    response.set_header("Expires", "0")
+    response.content_type = "application/json"
+    return response
+
+
 @app.get("/benchmark_output/<filename:path>")
 def serve_benchmark_output(filename):
     response = static_file(filename, root=app.config["helm.outputpath"])
@@ -36,6 +70,14 @@ def serve_benchmark_output(filename):
     return response
 
 
+@app.get("/cache/output/<filename:path>")
+def serve_cache_output(filename):
+    response = static_file(filename, root=app.config["helm.cacheoutputpath"])
+    response.set_header("Cache-Control", "no-cache, no-store, must-revalidate")
+    response.set_header("Expires", "0")
+    return response
+
+
 @app.get("/")
 @app.get("/<filename:path>")
 def serve_static(filename="index.html"):
@@ -53,6 +95,12 @@ def main():
         help="The location of the output path (filesystem path or URL)",
         default="benchmark_output",
     )
+    parser.add_argument(
+        "--cache-output-path",
+        type=str,
+        help="The location of the filesystem cache output folder (filesystem path or URL)",
+        default="prod_env/cache/output",
+    )
     parser.add_argument(
         "--suite",
         type=str,
@@ -65,6 +113,11 @@ def main():
         default=None,
         help="Experimental: The release to serve. If unset, don't serve a release, and serve the latest suite instead.",
     )
+    parser.add_argument(
+        "--jquery",
+        action="store_true",
+        help="Whether to serve the legacy jQuery frontend instead of the React frontend.",
+    )
     args = parser.parse_args()
 
     if args.suite and args.release:
@@ -73,7 +126,8 @@ def main():
     # Determine the location of the static directory.
     # This is a hack: it assumes that the static directory has a physical location,
     # which is not always the case (e.g. when using zipimport).
-    resource_path = resources.files("helm.benchmark.static").joinpath("index.html")
+    static_package_name = "helm.benchmark.static" if args.jquery else "helm.benchmark.static_build"
+    resource_path = resources.files(static_package_name).joinpath("index.html")
     with resources.as_file(resource_path) as resource_filename:
         static_path = str(resource_filename.parent)
 
@@ -83,16 +137,19 @@ def main():
         # Output path is a URL, so set the output path base URL in the frontend to that URL
         # so that the frontend reads from that URL directly.
         app.config["helm.outputpath"] = None
+        # TODO: figure out helm.cacheoutputpath
         app.config["helm.outputurl"] = args.output_path
     else:
         # Output path is a location on disk, so set the output path base URL to /benchmark_output
         # and then serve files from the location on disk at that URL.
         app.config["helm.outputpath"] = path.abspath(args.output_path)
+        app.config["helm.cacheoutputpath"] = path.abspath(args.cache_output_path)
         app.config["helm.outputurl"] = "benchmark_output"
 
     app.config["helm.suite"] = args.suite or "latest"
     app.config["helm.release"] = args.release
 
+    print(f"After the web server has started, go to http://localhost:{args.port} to view your website.\n")
     app.run(host="0.0.0.0", port=args.port)
 
 

helm/benchmark/slurm_jobs.py

@@ -2,6 +2,8 @@ import re
 import subprocess
 from typing import Mapping, Set, Union
 
+from retrying import retry
+
 from helm.common.optional_dependencies import handle_module_not_found_error
 
 try:
@@ -67,6 +69,11 @@ def submit_slurm_job(command: str, slurm_args: Mapping[str, Union[str, int]]) ->
     return slurm.sbatch(command)
 
 
+@retry(
+    wait_incrementing_start=5 * 1000,  # 5 seconds
+    wait_incrementing_increment=5 * 1000,  # 5 seconds
+    stop_max_attempt_number=5,
+)
 def get_slurm_job_state(job_id: int) -> str:
     """Get the state of a Slurm job."""
     try:
@@ -80,6 +87,11 @@ def get_slurm_job_state(job_id: int) -> str:
     return search_result.group(1)
 
 
+@retry(
+    wait_incrementing_start=5 * 1000,  # 5 seconds
+    wait_incrementing_increment=5 * 1000,  # 5 seconds
+    stop_max_attempt_number=5,
+)
 def cancel_slurm_job(job_id: int) -> None:
     """Cancel a Slurm job."""
     try: