crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/truthful_qa_scenario.py

@@ -2,8 +2,19 @@ import csv
 import os
 from typing import List, Dict, Any
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
-from .scenario import Scenario, Instance, Reference, VALID_SPLIT, TRAIN_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    VALID_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class TruthfulQAScenario(Scenario):
@@ -68,7 +79,8 @@ class TruthfulQAScenario(Scenario):
         """Downloads the TruthfulQA dataset."""
         # Download the raw data
         data_dir = os.path.join(output_path, "data")
-        url = "https://raw.githubusercontent.com/sylinrl/TruthfulQA/main/TruthfulQA.csv"
+
+        url = "https://raw.githubusercontent.com/sylinrl/TruthfulQA/d71c110897f5d31c5d7f309e7bc316c152f6f031/data/v1/TruthfulQA.csv"  # noqa: E501
         ensure_directory_exists(data_dir)
         ensure_file_downloaded(source_url=url, target_path=os.path.join(data_dir, self.DATASET_FILE_NAME))
 
@@ -144,3 +156,15 @@ class TruthfulQAScenario(Scenario):
         valid_instances: List[Instance] = get_split_instances(VALID_SPLIT, data[split_k:])
 
         return train_instances + valid_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="truthful_qa",
+            display_name="TruthfulQA",
+            description="The TruthfulQA benchmarking for measuring model truthfulness and commonsense "
+            "knowledge in question answering [(Lin et al., "
+            "2022)](https://aclanthology.org/2022.acl-long.229/).",
+            taxonomy=TaxonomyInfo(task="question answering", what="?", when="?", who="?", language="English"),
+            main_metric="exact_match",
+            main_split="valid",
+        )

helm/benchmark/scenarios/tweetsentbr_scenario.py

@@ -0,0 +1,66 @@
+from typing import Any, List, Dict
+from pathlib import Path
+from datasets import load_dataset
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class TweetSentBRScenario(Scenario):
+    """
+    TweetSentBR is a corpus of Tweets in Brazilian Portuguese. It was labeled by several
+    annotators following steps stablished on the literature for improving reliability on
+    the task of Sentiment Analysis. Each Tweet was annotated in one of the three following classes:
+
+    Positive - tweets where a user meant a positive reaction or evaluation about the main topic on the post;
+    Negative - tweets where a user meant a negative reaction or evaluation about the main topic on the post;
+    Neutral - tweets not belonging to any of the last classes, usually not making a point, out of topic,
+    irrelevant, confusing or containing only objective data.
+
+    This dataset is a subset of the tweetSentBR, it contains only 75 samples from the training set
+    and all 2.000+ instances of the test set. This is meant for evaluating language models in a few-shot setting.
+    """
+
+    name = "simple_classification"
+    description = "Classify tweets into Positive, Negative or Neutral."
+    tags = ["classification"]
+
+    def process_dataset(self, dataset: Any, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        label_names = {"Positive": "Positivo", "Negative": "Negativo", "Neutral": "Neutro"}
+        for example in dataset[split]:
+            input = Input(text=example["sentence"])
+            # NOTE: For classification scenarios, the reference outputs should be the same
+            # for all instances, and should include both correct and incorrect classes.
+            # HELM only supports single-label classification. Exactly one reference
+            # should have the CORRECT_TAG tag.
+            references = [
+                Reference(Output(text=label_names[example["label"]]), tags=[CORRECT_TAG]),
+            ]
+            instance = Instance(input=input, references=references, split=split)
+            instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+        dataset = load_dataset("eduagarcia/tweetsentbr_fewshot", cache_dir=cache_dir)
+        splits: Dict[str, str] = {
+            "train": TRAIN_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for split in splits:
+            if split not in splits.keys():
+                hlog(f"{split} split doesn't exist, skipping")
+                continue
+            instances.extend(self.process_dataset(dataset, splits[split]))
+
+        return instances

helm/benchmark/scenarios/twitter_aae_scenario.py

@@ -2,9 +2,10 @@ import csv
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 CODALAB_URI_TEMPLATE: str = (
     "https://worksheets.codalab.org/rest/bundles/0x31485f8c37ad481fb9f4e9bf7ccff6e5/contents/blob/"
@@ -56,3 +57,21 @@ class TwitterAAEScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="twitter_aae",
+            display_name="TwitterAAE",
+            description="The TwitterAAE corpus of [Blodgett et al. "
+            "(2016)](https://aclanthology.org/D16-1120/) for measuring language model "
+            "performance in tweets as a function of speaker dialect.",
+            taxonomy=TaxonomyInfo(
+                task="language modeling",
+                what="?",
+                when="?",
+                who="?",
+                language="English (AAE-aligned and White-aligned)",
+            ),
+            main_metric="bits_per_byte",
+            main_split="test",
+        )

helm/benchmark/scenarios/unitxt_scenario.py

@@ -0,0 +1,62 @@
+from typing import List
+from datasets import load_dataset
+
+from helm.benchmark.scenarios.scenario import (
+    Output,
+    Reference,
+    Scenario,
+    Instance,
+    Input,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    VALID_SPLIT,
+)
+
+
+class UnitxtScenario(Scenario):
+    """Integration with Unitxt: https://unitxt.rtfd.io/"""
+
+    name = "unitxt"
+    description = "Unitxt Scenarios"
+    tags = ["unitxt"]
+
+    UNITXT_SPLIT_NAME_TO_HELM_SPLIT_NAME = {
+        "train": TRAIN_SPLIT,
+        "test": TEST_SPLIT,
+        "validation": VALID_SPLIT,
+    }
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.kwargs = kwargs
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        if len(self.kwargs) == 1 and "recipe" in self.kwargs:
+            dataset_name = self.kwargs["recipe"]
+        else:
+            dataset_name = ",".join(f"{key}={value}" for key, value in self.kwargs.items())
+        dataset = load_dataset("unitxt/data", dataset_name, trust_remote_code=True)
+
+        instances: List[Instance] = []
+
+        for unitxt_split_name, helm_split_name in UnitxtScenario.UNITXT_SPLIT_NAME_TO_HELM_SPLIT_NAME.items():
+            dataset_split = dataset.get(unitxt_split_name)
+            if dataset_split is None:
+                continue
+            for index, row in enumerate(dataset_split):
+                references = [
+                    Reference(
+                        output=Output(text=reference_text),
+                        tags=[CORRECT_TAG],
+                    )
+                    for reference_text in row["references"]
+                ]
+                instance = Instance(
+                    id=f"{unitxt_split_name}{index}",
+                    input=Input(text=row["source"]),
+                    references=references,
+                    split=helm_split_name,
+                )
+                instances.append(instance)
+        return instances

helm/benchmark/scenarios/verifiability_judgment_scenario.py

@@ -4,7 +4,7 @@ import json
 from typing import Dict, List, Tuple
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,
@@ -137,7 +137,9 @@ class VerifiabilityJudgementScenario(Scenario):
         for _, filesplit in split_to_filesplit.items():
             target_name = f"verifiability_judgments_{filesplit}.jsonl"
             target_path: str = os.path.join(data_path, target_name)
-            url: str = f"https://github.com/nelson-liu/evaluating-verifiability-in-generative-search-engines/raw/40bf37e3a4eca7d82515df2c800ec9605458d637/verifiability_judgments/{target_name}.gz"  # noqa: E501
+            url: str = (
+                f"https://github.com/nelson-liu/evaluating-verifiability-in-generative-search-engines/raw/40bf37e3a4eca7d82515df2c800ec9605458d637/verifiability_judgments/{target_name}.gz"  # noqa: E501
+            )
             ensure_file_downloaded(source_url=url, target_path=target_path)
             assert os.path.exists(target_path)
 

helm/benchmark/scenarios/vicuna_scenario.py

@@ -2,8 +2,9 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, ScenarioMetadata
 
 
 class VicunaScenario(Scenario):
@@ -26,7 +27,7 @@ class VicunaScenario(Scenario):
             return self.category == "all" or raw["category"] == self.category
 
         # Download the raw data
-        source_url = "https://raw.githubusercontent.com/lm-sys/FastChat/main/fastchat/eval/table/question.jsonl"
+        source_url = "https://raw.githubusercontent.com/lm-sys/FastChat/v0.2.5/fastchat/eval/table/question.jsonl"
         data_path: str = os.path.join(output_path, "vicuna_questions.jsonl")
 
         ensure_file_downloaded(
@@ -47,3 +48,22 @@ class VicunaScenario(Scenario):
                 )
                 instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="vicuna",
+            display_name="Vicuna",
+            short_display_name="Vicuna",
+            description="The set of prompts used by the "
+            "[Vicuna](https://lmsys.org/blog/2023-03-30-vicuna/) team to evaluate "
+            "instruction-following models.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Instructions for LLMs",
+                when="Before 2023",
+                who="Unknown",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/vision_language/a_okvqa_scenario.py

@@ -0,0 +1,83 @@
+import os
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    ALL_SPLITS,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+
+
+class AOKVQAScenario(Scenario):
+    """
+    A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense
+    and world knowledge to answer.
+
+    @misc{schwenk2022aokvqa,
+          title={A-OKVQA: A Benchmark for Visual Question Answering using World Knowledge},
+          author={Dustin Schwenk and Apoorv Khandelwal and Christopher Clark and Kenneth Marino and Roozbeh Mottaghi},
+          year={2022},
+          eprint={2206.01718},
+          archivePrefix={arXiv},
+          primaryClass={cs.CV}
+    }
+
+    Paper: https://arxiv.org/abs/2206.01718
+    Website: https://huggingface.co/datasets/HuggingFaceM4/A-OKVQA
+    """
+
+    HF_DATASET_NAME: str = "HuggingFaceM4/A-OKVQA"
+
+    name = "a_okvqa"
+    description = (
+        "A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of "
+        "commonsense and world knowledge to answer ([Schwenk et al., 2022](https://arxiv.org/abs/2206.01718))."
+    )
+    tags = ["vision-language", "knowledge", "reasoning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        instances: List[Instance] = []
+        for helm_split in ALL_SPLITS:
+            if helm_split == TEST_SPLIT:
+                # The examples in the test split does not have answers
+                continue
+
+            split = "validation" if helm_split == VALID_SPLIT else helm_split
+
+            for row in tqdm(load_dataset(self.HF_DATASET_NAME, cache_dir=output_path, split=split)):
+                image_filename: str = f"{row['question_id']}.jpg"
+                local_image_path: str = os.path.join(images_path, image_filename)
+                image = row["image"]
+                if not os.path.exists(local_image_path):
+                    image.save(local_image_path)
+
+                content: List[MediaObject] = [
+                    MediaObject(location=local_image_path, content_type="image/jpeg"),
+                    MediaObject(text=row["question"], content_type="text/plain"),
+                ]
+                instances.append(
+                    Instance(
+                        Input(multimedia_content=MultimediaObject(content)),
+                        references=[
+                            Reference(Output(text=choice), tags=[CORRECT_TAG] if i == row["correct_choice_idx"] else [])
+                            for i, choice in enumerate(row["choices"])
+                        ],
+                        split=helm_split,
+                    )
+                )
+
+        return instances

helm/benchmark/scenarios/vision_language/bingo_scenario.py

@@ -0,0 +1,103 @@
+import os.path
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class BingoScenario(Scenario):
+    """
+    Holistic Analysis of Hallucination in GPT-4V(ision): Bias and Interference Challenges
+
+    We introduce a new benchmark, namely, the Bias and Interference Challenges in Visual Language Models (Bingo).
+    This benchmark is designed to evaluate and shed light on the two common types of hallucinations in visual
+    language models: bias and interference. Here, bias refers to the model's tendency to hallucinate certain types
+    of responses, possibly due to imbalance in its training data. Interference pertains to scenarios where the
+    judgment of GPT-4V(ision) can be disrupted due to how the text prompt is phrased or how the input image is
+    presented. The benchmark consists of open-ended question-answer pairs, and we employ open-ended generation
+    metrics for evaluation. In the experiment, we identify a notable regional bias, whereby GPT-4V(ision) is
+    better at interpreting Western images or images with English writing compared to images from other countries
+    or containing text in other languages.
+
+
+    @article{cui2023holistic,
+    title={Holistic analysis of hallucination in gpt-4v (ision): Bias and interference challenges},
+    author={Cui, Chenhang and Zhou, Yiyang and Yang, Xinyu and Wu, Shirley and Zhang, Linjun and
+    Zou, James and Yao, Huaxiu},
+    journal={arXiv preprint arXiv:2311.03287},
+    year={2023}
+    }
+
+    Paper: https://arxiv.org/abs/2311.03287
+    """
+
+    BINGO_HUGGINGFACE_DATASET_URL: str = "https://huggingface.co/datasets/PahaII/Bingo/resolve/main"
+
+    IMAGE_URL: str = "https://huggingface.co/datasets/PahaII/Bingo/resolve/main/images/{image_path}?download=true"
+
+    SUBJECTS: List[str] = ["T2I", "I2I", "OCR", "Factual", "Region"]
+
+    name = "bingo"
+    description = (
+        "Evaluate multimodal models on biased and inference-challenging scenarios with five subjects "
+        "([Cui et al., 2023](https://arxiv.org/abs/2311.03287))."
+    )
+    tags = ["vision-language"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        # There is only the test split in Unicorn benchmark
+        instances: List[Instance] = []
+        question_data_files = {TEST_SPLIT: f"{self.BINGO_HUGGINGFACE_DATASET_URL}/{self._subject}.json"}
+
+        # Process the test set
+        for row in tqdm(
+            load_dataset(
+                "json",
+                data_files=question_data_files,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            # Download the image
+            image_path: str = row["image_path"]
+            local_image_path: str = os.path.join(output_path, image_path)
+            ensure_file_downloaded(
+                source_url=self.IMAGE_URL.format(image_path=image_path),
+                target_path=local_image_path,
+                unpack=False,
+            )
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type="image/png"),
+                MediaObject(text=row["question"], content_type="text/plain"),
+            ]
+            answer: str = row["answer"]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/blink_scenario.py

@@ -0,0 +1,140 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.images_utils import generate_hash
+
+
+class BlinkScenario(Scenario):
+    """
+    BLINK is a benchmark containing 14 visual perception tasks that can be solved by humans “within a blink”,
+    but pose significant challenges for VLMs.
+
+    Website: https://zeyofu.github.io/blink/
+
+    @article{fu2024blink,
+        title={BLINK: Multimodal Large Language Models Can See but Not Perceive},
+        author={Fu, Xingyu and Hu, Yushi and Li, Bangzheng and Feng, Yu and Wang, Haoyu and Lin, Xudong and Roth,
+        Dan and Smith, Noah A and Ma, Wei-Chiu and Krishna, Ranjay},
+        journal={arXiv preprint arXiv:2404.12390},
+        year={2024}
+    }
+    """
+
+    HUGGINGFACE_DATASET_NAME: str = "BLINK-Benchmark/BLINK"
+
+    VALID_CATEGORIES: List[str] = [
+        "Art_Style",
+        "Counting",
+        "Forensic_Detection",
+        "Functional_Correspondence",
+        "IQ_Test",
+        "Jigsaw",
+        "Multi-view_Reasoning",
+        "Object_Localization",
+        "Relative_Depth",
+        "Relative_Reflectance",
+        "Semantic_Correspondence",
+        "Spatial_Relation",
+        "Visual_Correspondence",
+        "Visual_Similarity",
+    ]
+
+    name = "blink"
+    description = (
+        "BLINK is a benchmark containing 14 visual perception tasks that can be solved by humans within a blink, "
+        "but pose significant challenges for VLMs. ([Fu, 2024](https://arxiv.org/abs/2404.12390))."
+    )
+    tags = ["vision-language", "knowledge", "reasoning"]
+
+    def __init__(self, category: str):
+        super().__init__()
+
+        if category not in self.VALID_CATEGORIES:
+            raise ValueError(f"Invalid category: {category}. Valid categories are: {self.VALID_CATEGORIES}")
+        self._category: str = category
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        def save_image(image) -> str:
+            image_file_name: str = generate_hash(image) + ".jpg"
+            local_image_path: str = os.path.join(output_path, image_file_name)
+            if not os.path.exists(local_image_path):
+                image.save(local_image_path)
+            return local_image_path
+
+        def get_image_header(image_index: int) -> str:
+            if image_index == 1:
+                return "First image:"
+            elif image_index == 2:
+                return "Second image:"
+            elif image_index == 3:
+                return "Third image:"
+            elif image_index == 4:
+                return "Fourth image:"
+            else:
+                raise ValueError(f"Invalid image index: {image_index}")
+
+        instances: List[Instance] = []
+        for row in tqdm(
+            load_dataset(self.HUGGINGFACE_DATASET_NAME, self._category, split="val", cache_dir=output_path)
+        ):
+            # Save the image(s) to disk
+            has_multiple_images: bool = row["image_2"] is not None
+            content: List[MediaObject] = []
+
+            if has_multiple_images:
+                # An example can have up to 4 images
+                for i in range(1, 5):
+                    image_i = row[f"image_{i}"]
+                    if image_i is None:
+                        break
+
+                    # Before each image, include a header text that indicates which number image it is.
+                    # Some prompts refer to specific image numbers within the question, e.g.,
+                    # "Given three similar but different images, take the first image as reference.
+                    # Can you tell which one of the latter two images is most similar to the first one?
+                    # Select from the following choices. (A) the second image (B) the third image"
+                    image_path: str = save_image(image_i)
+                    content.extend(
+                        [
+                            MediaObject(text=get_image_header(i), content_type="text/plain"),
+                            MediaObject(location=image_path, content_type="image/jpeg"),
+                        ]
+                    )
+            else:
+                image1 = row["image_1"]
+                image1_path: str = save_image(image1)
+                content.append(MediaObject(location=image1_path, content_type="image/jpeg"))
+
+            # Add the prompt that has both the question and the answer choices
+            prompt: str = row["prompt"]
+            # Replace (A), (B), (C), (D) with \nA. \nB. \nC. \nD. since we are just expecting the letter answer
+            prompt = prompt.replace("(A)", "\nA.").replace("(B)", "\nB.").replace("(C)", "\nC.").replace("(D)", "\nD.")
+            content.append(MediaObject(text=prompt, content_type="text/plain"))
+
+            # The answer has the correct letter choices surrounded by parentheses
+            paren_letter_answer: str = row["answer"]
+            assert (
+                paren_letter_answer[0] == "(" and paren_letter_answer[-1] == ")"
+            ), f"Unexpected answer format: {paren_letter_answer}"
+            letter_answer: str = paren_letter_answer[1]
+            references: List[Reference] = [
+                Reference(output=Output(text=letter_answer), tags=[CORRECT_TAG]),
+            ]
+            instances.append(
+                Instance(Input(multimedia_content=MultimediaObject(content)), references=references, split=VALID_SPLIT)
+            )
+
+        return instances