crfm-helm 0.5.1__py3-none-any.whl → 0.5.3__py3-none-any.whl

helm/benchmark/scenarios/vision_language/unicorn_scenario.py

@@ -40,7 +40,7 @@ class UnicornScenario(Scenario):
     Paper: https://arxiv.org/abs/2311.16101
     """
 
-    UNICORN_HUGGINGFACE_DATASET_NAME: str = "PahaII/unicorn"
+    UNICORN_HUGGINGFACE_DATASET_URL: str = "https://huggingface.co/datasets/PahaII/unicorn/resolve/main"
 
     IMAGE_URL: str = "https://huggingface.co/datasets/PahaII/unicorn/resolve/main/images/{image_path}?download=true"
 
@@ -55,8 +55,8 @@ class UnicornScenario(Scenario):
 
     name = "unicorn"
     description = (
-        "Evaluate multimodal models on two out-of-distribution scenarios with four subjects"
-        " ([paper](https://arxiv.org/abs/2311.16101))."
+        "Evaluate multimodal models on two out-of-distribution scenarios with four subjects "
+        "([Tu et al., 2023](https://arxiv.org/abs/2311.16101))."
     )
     tags = ["vision-language"]
 
@@ -72,12 +72,12 @@ class UnicornScenario(Scenario):
 
         # There is only the test split in Unicorn benchmark
         instances: List[Instance] = []
-        question_data_files = {TEST_SPLIT: f"{self._subject}.json"}
+        question_data_files = {TEST_SPLIT: f"{self.UNICORN_HUGGINGFACE_DATASET_URL}/{self._subject}.json"}
 
         # Process the test set
         for row in tqdm(
             load_dataset(
-                self.UNICORN_HUGGINGFACE_DATASET_NAME,
+                "json",
                 data_files=question_data_files,
                 split=TEST_SPLIT,
                 cache_dir=output_path,

helm/benchmark/scenarios/vision_language/vibe_eval_scenario.py

@@ -0,0 +1,98 @@
+import os.path
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+
+
+class VibeEvalScenario(Scenario):
+    """
+    Vibe-Eval: A hard evaluation suite for measuring progress of multimodal language models
+
+    We introduce Vibe-Eval: a new open benchmark and framework for evaluating multimodal chat
+    models. Vibe-Eval consists of 269 visual understanding prompts, including 100 of hard
+    difficulty, complete with gold-standard responses authored by experts. Vibe-Eval is
+    open-ended and challenging with dual objectives: (i) vibe checking multimodal chat models
+    for day-to-day tasks and (ii) rigorously testing and probing the capabilities of present
+    frontier models. Notably, our hard set contains >50% questions that all frontier models
+    answer incorrectly. We also discuss trade-offs between human and automatic evaluation,
+    and show that automatic model evaluation using Reka Core roughly correlates to human judgment.
+
+    @article{padlewski2024vibe,
+    title={Vibe-Eval: A hard evaluation suite for measuring progress of multimodal language models},
+    author={Padlewski, Piotr and Bain, Max and Henderson, Matthew and Zhu, Zhongkai
+    and Relan, Nishant and Pham, Hai and Ong, Donovan and Aleksiev, Kaloyan and Ormazabal, Aitor
+    and Phua, Samuel and others},
+    journal={arXiv preprint arXiv:2405.02287},
+    year={2024}
+    }
+
+    Paper: https://arxiv.org/abs/2405.02287
+    """
+
+    VIBE_EVAL_HUGGINGFACE_DATASET_NAME: str = "RekaAI/VibeEval"
+
+    SUBJECTS: List[str] = [
+        "difficulty-hard",
+        "difficulty-normal",
+    ]
+
+    name = "vibe_eval"
+    description = (
+        "Evaluate multimodal models on day-to-day tasks "
+        "([Padlewski et al., 2024](https://arxiv.org/abs/2405.02287))."
+    )
+    tags = ["vision-language", "knowledge", "reasoning"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        instances: List[Instance] = []
+        # Process the test set
+        for row in tqdm(
+            load_dataset(
+                self.VIBE_EVAL_HUGGINGFACE_DATASET_NAME,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            if row["category"] != self._subject:
+                continue
+            example_id: str = row["example_id"].replace("/", "-")
+            # Save the image locally
+            local_image_path: str = os.path.join(images_path, f"{example_id}.png")
+            if not os.path.exists(local_image_path):
+                row["image"].convert("RGB").save(local_image_path, "PNG", optimize=True)
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type="image/png"),
+                MediaObject(text=row["prompt"], content_type="text/plain"),
+            ]
+            answer: str = row["reference"]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/viz_wiz_scenario.py

@@ -53,7 +53,7 @@ class VizWizScenario(Scenario):
     name = "viz_wiz"
     description = (
         "Real-world VQA dataset consisting of questions asked by "
-        "people who are blind ([paper](https://arxiv.org/abs/1802.08218))."
+        "people who are blind ([Gurari et al., 2018](https://arxiv.org/abs/1802.08218))."
     )
     tags = ["vision-language", "visual question answering"]
 

helm/benchmark/scenarios/vision_language/vqa_scenario.py

@@ -55,7 +55,9 @@ class VQAScenario(Scenario):
     }
 
     name = "vqa"
-    description = "Open-ended questions about images ([paper](https://arxiv.org/abs/1612.00837))."
+    description = (
+        "Open-ended questions about real-world images " "([Goyal et al., 2017](https://arxiv.org/abs/1612.00837))."
+    )
     tags = ["vision-language", "visual question answering"]
 
     def get_instances(self, output_path: str) -> List[Instance]:

helm/benchmark/scenarios/xstest_scenario.py

@@ -0,0 +1,35 @@
+from typing import List
+from datasets import load_dataset
+
+from .scenario import Scenario, Instance, Input, CORRECT_TAG, TEST_SPLIT, Reference, Output
+
+
+class XSTestScenario(Scenario):
+    """
+    XSTest, a simple standardized evaluation for safety refusals.
+    """
+
+    name = "XSTest"
+    description = "XSTest dataset - safety benchmark"
+    tags = ["safety"]
+
+    def __init__(self, subject: str = "all"):
+        super().__init__()
+        self.subset = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # read explicit data
+        dataset = load_dataset("walledai/XSTest", split="test", revision="f1d713187c61b6ae64e602d74f0b3d812cc2e8e8")
+
+        # Read all the instances
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=row["prompt"])
+            references = []
+            for column_name in ["focus", "type", "note"]:
+                if row[column_name]:
+                    references += [Reference(output=Output(text=row[column_name]), tags=[])]
+            references += [Reference(output=Output(text=row["label"]), tags=[CORRECT_TAG])]
+            instance = Instance(input=input, references=references, split=TEST_SPLIT)
+            instances.append(instance)
+        return instances

helm/benchmark/server.py

@@ -113,11 +113,6 @@ def main():
         default=None,
         help="Experimental: The release to serve. If unset, don't serve a release, and serve the latest suite instead.",
     )
-    parser.add_argument(
-        "--jquery",
-        action="store_true",
-        help="Whether to serve the legacy jQuery frontend instead of the React frontend.",
-    )
     args = parser.parse_args()
 
     if args.suite and args.release:
@@ -126,7 +121,7 @@ def main():
     # Determine the location of the static directory.
     # This is a hack: it assumes that the static directory has a physical location,
     # which is not always the case (e.g. when using zipimport).
-    static_package_name = "helm.benchmark.static" if args.jquery else "helm.benchmark.static_build"
+    static_package_name = "helm.benchmark.static_build"
     resource_path = resources.files(static_package_name).joinpath("index.html")
     with resources.as_file(resource_path) as resource_filename:
         static_path = str(resource_filename.parent)