crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/benchmark/scenarios/thai_exam_scenario.py

@@ -0,0 +1,135 @@
+import os
+from typing import Dict, List
+import json
+
+from helm.common.general import ensure_file_downloaded
+from helm.common.hierarchical_logger import hlog
+from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+
+
+class ThaiExamScenario(Scenario):
+    """
+    ThaiExam, a benchmark comprising Thai multiple-choice examinations as follows:
+
+    ∙ ONET: The Ordinary National Educational Test (ONET) is an examination for students in Thailand.
+    We select the grade-12 ONET exam, which comprises 5 subjects and each question has 5 choices.
+    These subjects are Thai, English, Mathematics, Social Studies, and Science.
+    Amounting to a total of 170 questions and options.
+
+    ∙ IC: The Investment Consultant (IC) examination, a licensing test for investment professionals in Thailand.
+    Developed by the Stock Exchange of Thailand (SET), features 4 choices per question.
+    We extracted questions for levels 1, 2, and 3 resulting in a total of 95 questions and options.
+
+    ∙ TGAT: The Thai General Aptitude Test (TGAT), a national high school examination in Thailand.
+    Focuses on critical and logical thinking skills.
+    We collected a total of 90 questions and answers. The TGAT consists of four choices per question.
+
+    ∙ TPAT-1: The Thai Professional Aptitude Test 1 (TPAT-1) is a national high school examination in Thailand.
+    The Exam assesses students’ professional skills requirement in medical schools.
+    This subset contains reasoning and medical ethics. We collected a total of 116 questions and answers.
+    The TPAT-1 consists of 5 choices per question.
+
+    ∙ A-Level: An academic knowledge assessment examination (Applied Knowledge Level)
+    that covers general foundational subjects taught in schools.
+    The content assessed in this examination aligns with the curriculum guidelines
+    and emphasizes the practical application of knowledge in daily life.
+    We collected a total of 175 questions and answers.
+
+    We created and used these exams to evaluate the performance of the Typhoon models(https://arxiv.org/abs/2312.13951).
+
+    Prompt models using the following format
+
+        <input>                  # train
+        A. <reference>
+        B. <reference>
+        C. <reference>
+        D. <reference>
+        E. <reference>
+        Answer: <A/B/C/D/E>
+
+        x N (N-shot)
+
+        <input>                  # test
+        A. <reference1>
+        B. <reference2>
+        C. <reference3>
+        D. <reference4>
+        E. <reference5>
+        Answer:
+
+    For example:
+
+        ในระบบย่อยอาหารของมนุษย์ การดูดซึมสารอาหารส่วนใหญ่เกิดขึ้นที่อวัยวะใด?
+        A. ลำไส้เล็ก
+        B. ตับอ่อน
+        C. ลำไส้ใหญ่
+        D. กระเพาะอาหาร
+        E. หัวใจ
+        Answer: A
+
+        ข้อใดอธิบายเกี่ยวกับแรงไฟฟ้าได้ถูกต้อง?
+        A. เกิดได้โดยที่วัตถุไม่ต้องสัมผัสกัน
+        B. เป็นได้เฉพาะแรงผลักเท่านั้น
+        C. เป็นได้เฉพาะแรงดูดเท่านั้น
+        D. เป็นแรงต้านระหว่างวัตถุเท่านั้น
+        E. ถูกทุกข้อ
+        Answer:
+
+    Target: A
+    """
+
+    name = "thai_exam"
+    description = "ThaiExam benchmark comprising Thai multiple-choice examinations."
+    tags = ["knowledge", "multiple_choice"]
+
+    def __init__(self, exam: str):
+        super().__init__()
+        self.exam = exam
+
+    def download_thai_exam(self, path: str):
+        ensure_file_downloaded(
+            "https://storage.googleapis.com/thai_dataset/thai_exam.tar.gz",
+            target_path=path,
+            unpack=True,
+        )
+
+    def process_jsonl(self, jsonl_path: str, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        hlog(f"Reading {jsonl_path}")
+        with open(jsonl_path, "r") as f:
+            for line in f:
+                data = json.loads(line)
+                # for handle missing key incase of some subject doesn't have all 5 choices
+                answers = [data[key] for key in ["a", "b", "c", "d", "e"] if key in data and data[key] != ""]
+                answers_dict = dict(zip(["A", "B", "C", "D", "E"], answers))
+
+                question, correct_answer = data["question"], answers_dict[data["answer"].upper()]
+
+                def answer_to_reference(answer: str) -> Reference:
+                    return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+                instance = Instance(
+                    input=Input(text=question),
+                    references=list(map(answer_to_reference, answers)),
+                    split=split,
+                )
+                instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "data")
+        self.download_thai_exam(data_path)
+
+        instances: List[Instance] = []
+        splits: Dict[str, str] = {
+            "train": TRAIN_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for split in splits:
+            jsonl_path: str = os.path.join(data_path, self.exam, f"{self.exam}_{split}.jsonl")
+            if not os.path.exists(jsonl_path):
+                hlog(f"{jsonl_path} doesn't exist, skipping")
+                continue
+            instances.extend(self.process_jsonl(jsonl_path, splits[split]))
+
+        return instances

helm/benchmark/scenarios/unitxt_scenario.py

@@ -0,0 +1,56 @@
+from typing import List
+from datasets import load_dataset
+
+from helm.benchmark.scenarios.scenario import (
+    Output,
+    Reference,
+    Scenario,
+    Instance,
+    Input,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    VALID_SPLIT,
+)
+
+
+class UnitxtScenario(Scenario):
+    """Integration with Unitxt: https://unitxt.rtfd.io/"""
+
+    name = "unitxt"
+    description = "Unitxt Scenarios"
+    tags = ["unitxt"]
+
+    UNITXT_SPLIT_NAME_TO_HELM_SPLIT_NAME = {
+        "train": TRAIN_SPLIT,
+        "test": TEST_SPLIT,
+        "validation": VALID_SPLIT,
+    }
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.kwargs = kwargs
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset_name = ",".join(f"{key}={value}" for key, value in self.kwargs.items())
+        dataset = load_dataset("unitxt/data", dataset_name, trust_remote_code=True)
+
+        instances: List[Instance] = []
+
+        for unitxt_split_name, helm_split_name in UnitxtScenario.UNITXT_SPLIT_NAME_TO_HELM_SPLIT_NAME.items():
+            for index, row in enumerate(dataset[unitxt_split_name]):
+                references = [
+                    Reference(
+                        output=Output(text=reference_text),
+                        tags=[CORRECT_TAG],
+                    )
+                    for reference_text in row["references"]
+                ]
+                instance = Instance(
+                    id=f"{unitxt_split_name}{index}",
+                    input=Input(text=row["source"]),
+                    references=references,
+                    split=helm_split_name,
+                )
+                instances.append(instance)
+        return instances

helm/benchmark/scenarios/verifiability_judgment_scenario.py

@@ -137,7 +137,9 @@ class VerifiabilityJudgementScenario(Scenario):
         for _, filesplit in split_to_filesplit.items():
             target_name = f"verifiability_judgments_{filesplit}.jsonl"
             target_path: str = os.path.join(data_path, target_name)
-            url: str = f"https://github.com/nelson-liu/evaluating-verifiability-in-generative-search-engines/raw/40bf37e3a4eca7d82515df2c800ec9605458d637/verifiability_judgments/{target_name}.gz"  # noqa: E501
+            url: str = (
+                f"https://github.com/nelson-liu/evaluating-verifiability-in-generative-search-engines/raw/40bf37e3a4eca7d82515df2c800ec9605458d637/verifiability_judgments/{target_name}.gz"  # noqa: E501
+            )
             ensure_file_downloaded(source_url=url, target_path=target_path)
             assert os.path.exists(target_path)
 

helm/benchmark/scenarios/vicuna_scenario.py

@@ -26,7 +26,7 @@ class VicunaScenario(Scenario):
             return self.category == "all" or raw["category"] == self.category
 
         # Download the raw data
-        source_url = "https://raw.githubusercontent.com/lm-sys/FastChat/main/fastchat/eval/table/question.jsonl"
+        source_url = "https://raw.githubusercontent.com/lm-sys/FastChat/v0.2.5/fastchat/eval/table/question.jsonl"
         data_path: str = os.path.join(output_path, "vicuna_questions.jsonl")
 
         ensure_file_downloaded(

helm/benchmark/scenarios/vision_language/a_okvqa_scenario.py

@@ -0,0 +1,83 @@
+import os
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    ALL_SPLITS,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+
+
+class AOKVQAScenario(Scenario):
+    """
+    A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of commonsense
+    and world knowledge to answer.
+
+    @misc{schwenk2022aokvqa,
+          title={A-OKVQA: A Benchmark for Visual Question Answering using World Knowledge},
+          author={Dustin Schwenk and Apoorv Khandelwal and Christopher Clark and Kenneth Marino and Roozbeh Mottaghi},
+          year={2022},
+          eprint={2206.01718},
+          archivePrefix={arXiv},
+          primaryClass={cs.CV}
+    }
+
+    Paper: https://arxiv.org/abs/2206.01718
+    Website: https://huggingface.co/datasets/HuggingFaceM4/A-OKVQA
+    """
+
+    HF_DATASET_NAME: str = "HuggingFaceM4/A-OKVQA"
+
+    name = "a_okvqa"
+    description = (
+        "A crowdsourced dataset composed of a diverse set of about 25K questions requiring a broad base of "
+        "commonsense and world knowledge to answer ([paper](https://arxiv.org/abs/2206.01718))."
+    )
+    tags = ["vision-language", "knowledge", "reasoning"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        instances: List[Instance] = []
+        for helm_split in ALL_SPLITS:
+            if helm_split == TEST_SPLIT:
+                # The examples in the test split does not have answers
+                continue
+
+            split = "validation" if helm_split == VALID_SPLIT else helm_split
+
+            for row in tqdm(load_dataset(self.HF_DATASET_NAME, cache_dir=output_path, split=split)):
+                image_filename: str = f"{row['question_id']}.jpg"
+                local_image_path: str = os.path.join(images_path, image_filename)
+                image = row["image"]
+                if not os.path.exists(local_image_path):
+                    image.save(local_image_path)
+
+                content: List[MediaObject] = [
+                    MediaObject(location=local_image_path, content_type="image/jpeg"),
+                    MediaObject(text=row["question"], content_type="text/plain"),
+                ]
+                instances.append(
+                    Instance(
+                        Input(multimedia_content=MultimediaObject(content)),
+                        references=[
+                            Reference(Output(text=choice), tags=[CORRECT_TAG] if i == row["correct_choice_idx"] else [])
+                            for i, choice in enumerate(row["choices"])
+                        ],
+                        split=helm_split,
+                    )
+                )
+
+        return instances

helm/benchmark/scenarios/vision_language/bingo_scenario.py

@@ -0,0 +1,103 @@
+import os.path
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class BingoScenario(Scenario):
+    """
+    Holistic Analysis of Hallucination in GPT-4V(ision): Bias and Interference Challenges
+
+    We introduce a new benchmark, namely, the Bias and Interference Challenges in Visual Language Models (Bingo).
+    This benchmark is designed to evaluate and shed light on the two common types of hallucinations in visual
+    language models: bias and interference. Here, bias refers to the model's tendency to hallucinate certain types
+    of responses, possibly due to imbalance in its training data. Interference pertains to scenarios where the
+    judgment of GPT-4V(ision) can be disrupted due to how the text prompt is phrased or how the input image is
+    presented. The benchmark consists of open-ended question-answer pairs, and we employ open-ended generation
+    metrics for evaluation. In the experiment, we identify a notable regional bias, whereby GPT-4V(ision) is
+    better at interpreting Western images or images with English writing compared to images from other countries
+    or containing text in other languages.
+
+
+    @article{cui2023holistic,
+    title={Holistic analysis of hallucination in gpt-4v (ision): Bias and interference challenges},
+    author={Cui, Chenhang and Zhou, Yiyang and Yang, Xinyu and Wu, Shirley and Zhang, Linjun and
+    Zou, James and Yao, Huaxiu},
+    journal={arXiv preprint arXiv:2311.03287},
+    year={2023}
+    }
+
+    Paper: https://arxiv.org/abs/2311.03287
+    """
+
+    BINGO_HUGGINGFACE_DATASET_NAME: str = "PahaII/Bingo"
+
+    IMAGE_URL: str = "https://huggingface.co/datasets/PahaII/Bingo/resolve/main/images/{image_path}?download=true"
+
+    SUBJECTS: List[str] = ["T2I", "I2I", "OCR", "Factual", "Region"]
+
+    name = "bingo"
+    description = (
+        "Evaluate multimodal models on biased and inference-challenging scenarios with five subjects"
+        " ([paper](https://arxiv.org/abs/2311.03287))."
+    )
+    tags = ["vision-language"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        # There is only the test split in Unicorn benchmark
+        instances: List[Instance] = []
+        question_data_files = {TEST_SPLIT: f"{self._subject}.json"}
+
+        # Process the test set
+        for row in tqdm(
+            load_dataset(
+                self.BINGO_HUGGINGFACE_DATASET_NAME,
+                data_files=question_data_files,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            # Download the image
+            image_path: str = row["image_path"]
+            local_image_path: str = os.path.join(output_path, image_path)
+            ensure_file_downloaded(
+                source_url=self.IMAGE_URL.format(image_path=image_path),
+                target_path=local_image_path,
+                unpack=False,
+            )
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type="image/png"),
+                MediaObject(text=row["question"], content_type="text/plain"),
+            ]
+            answer: str = row["answer"]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/crossmodal_3600_scenario.py

@@ -0,0 +1,134 @@
+import json
+import os
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+
+
+class Crossmodal3600Scenario(Scenario):
+    """
+    Crossmodal-3600 dataset (XM3600 in short), a geographically-diverse set of 3600 images annotated
+    with human-generated reference captions in 36 languages.
+
+    @inproceedings{ThapliyalCrossmodal2022,
+      author        = {Ashish Thapliyal and Jordi Pont-Tuset and Xi Chen and Radu Soricut},
+      title         = {{Crossmodal-3600: A Massively Multilingual Multimodal Evaluation Dataset}},
+      booktitle     = {EMNLP},
+      year          = {2022}
+    }
+
+    Paper: https://arxiv.org/abs/2205.12522
+    Website: https://google.github.io/crossmodal-3600/
+    """
+
+    LANGUAGE_TO_ID: Dict[str, str] = {
+        "arabic": "ar",
+        "bengali": "bn",
+        "chinese": "zh",
+        "croatian": "hr",
+        "cusco_quechua": "quz",
+        "czech": "cs",
+        "danish": "da",
+        "dutch": "nl",
+        "english": "en",
+        "persian": "fa",
+        "finnish": "fi",
+        "french": "fr",
+        "german": "de",
+        "greek": "el",
+        "hebrew": "he",
+        "hindi": "hi",
+        "hungarian": "hu",
+        "indonesian": "id",
+        "italian": "it",
+        "japanese": "ja",
+        "korean": "ko",
+        "maori": "mi",
+        "norwegian": "no",
+        "polish": "pl",
+        "portuguese": "pt",
+        "romanian": "ro",
+        "russian": "ru",
+        "spanish": "es",
+        "swahili": "sw",
+        "swedish": "sv",
+        "telugu": "te",
+        "thai": "th",
+        "turkish": "tr",
+        "ukrainian": "uk",
+        "vietnamese": "vi",
+    }
+
+    IMAGES_URL: str = "https://open-images-dataset.s3.amazonaws.com/crossmodal-3600/images.tgz"
+    CAPTIONS_URL: str = "https://google.github.io/crossmodal-3600/web-data/captions.zip"
+
+    name = "crossmodal_3600"
+    description = (
+        "Crossmodal-3600 dataset (XM3600 in short), a geographically-diverse set of 3600 images annotated "
+        "with human-generated reference captions in 36 languages. ([paper](https://arxiv.org/abs/2205.12522))."
+    )
+    tags = ["vision-language", "multilinguality"]
+
+    def __init__(self, location: str, language: str):
+        super().__init__()
+        self._locale_id: str = self.LANGUAGE_TO_ID[location]
+        self._language_id: str = self.LANGUAGE_TO_ID[language]
+        self._instruction: str = f"Generate a short caption for the following image in {language}."
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_file_downloaded(
+            source_url=self.IMAGES_URL,
+            target_path=images_path,
+            unpack=True,
+            unpack_type="untar",
+        )
+
+        captions_path: str = os.path.join(output_path, "captions.jsonl")
+        ensure_file_downloaded(
+            source_url=self.CAPTIONS_URL,
+            target_path=captions_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+
+        instances: List[Instance] = []
+        with open(captions_path, "r") as captions_file:
+            for line in captions_file:
+                example: Dict = json.loads(line)
+
+                locale_id: str = example["image/locale"]
+                if locale_id != self._locale_id:
+                    continue
+
+                key: str = example["image/key"]
+                image_path: str = os.path.join(images_path, f"{key}.jpg")
+                assert os.path.exists(image_path), f"Image {image_path} does not exist"
+
+                assert self._language_id in example, f"Language {self._language_id} not found in example"
+                all_captions: Dict = example[self._language_id]
+                captions: List[str] = all_captions["caption"]
+
+                content: List[MediaObject] = [
+                    MediaObject(text=self._instruction, content_type="text/plain"),
+                    MediaObject(location=image_path, content_type="image/jpeg"),
+                ]
+                instances.append(
+                    Instance(
+                        Input(multimedia_content=MultimediaObject(content)),
+                        references=[Reference(Output(text=caption), tags=[CORRECT_TAG]) for caption in captions],
+                        split=TEST_SPLIT,
+                    )
+                )
+
+        return instances

helm/benchmark/scenarios/vision_language/flickr30k_scenario.py

@@ -0,0 +1,74 @@
+import os
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists
+
+
+class Flickr30KScenario(Scenario):
+    """
+    An image caption corpus consisting of 158,915 crowdsourced captions describing 31,783 Flickr images.
+
+    @article{young2014image,
+      title={From image descriptions to visual denotations: New similarity metrics for semantic
+             inference over event descriptions},
+      author={Young, Peter and Lai, Alice and Hodosh, Micah and Hockenmaier, Julia},
+      journal={Transactions of the Association for Computational Linguistics},
+      volume={2},
+      pages={67--78},
+      year={2014},
+      publisher={MIT Press}
+    }
+
+    Paper: https://shannon.cs.illinois.edu/DenotationGraph/TACLDenotationGraph.pdf
+    Website: https://shannon.cs.illinois.edu/DenotationGraph/
+    """
+
+    HF_DATASET_NAME: str = "nlphuji/flickr30k"
+
+    name = "flickr30k"
+    description = (
+        "An image caption corpus consisting of 158,915 crowd-sourced captions describing 31,783 Flickr "
+        "images ([paper](https://shannon.cs.illinois.edu/DenotationGraph/TACLDenotationGraph.pdf))."
+    )
+    tags = ["vision-language"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        instances: List[Instance] = []
+        for row in tqdm(load_dataset(self.HF_DATASET_NAME, cache_dir=output_path, split="test")):
+            split: str = row["split"]
+            helm_split: str = VALID_SPLIT if split == "val" else split
+
+            image_filename: str = row["filename"]
+            local_image_path: str = os.path.join(images_path, image_filename)
+            image = row["image"]
+            if not os.path.exists(local_image_path):
+                image.save(local_image_path)
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type="image/jpeg"),
+            ]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=caption), tags=[CORRECT_TAG]) for caption in row["caption"]],
+                    split=helm_split,
+                )
+            )
+
+        return instances