crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/scenarios/test_simple_scenarios.py

@@ -0,0 +1,50 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.simple_scenarios import (
+    SimpleMCQAScenario,
+    SimpleShortAnswerQAScenario,
+    SimpleClassificationScenario,
+)
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_simple_mcqa_scenario():
+    scenario = SimpleMCQAScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 90
+    assert instances[0].input == Input(text="Is 10 even or odd?")
+    assert instances[0].references == [
+        Reference(output=Output(text="Even"), tags=[CORRECT_TAG]),
+        Reference(output=Output(text="Odd"), tags=[]),
+    ]
+    assert instances[0].split == "train"
+
+
+@pytest.mark.scenarios
+def test_simple_short_answer_qa_scenario():
+    scenario = SimpleShortAnswerQAScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 90
+    assert instances[0].input == Input(text="Is 10 even or odd?")
+    assert instances[0].references == [
+        Reference(output=Output(text="Even"), tags=[CORRECT_TAG]),
+    ]
+    assert instances[0].split == "train"
+
+
+@pytest.mark.scenarios
+def test_simple_classification_scenario():
+    scenario = SimpleClassificationScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = scenario.get_instances(tmpdir)
+    assert len(instances) == 90
+    assert instances[0].input == Input(text="10")
+    assert instances[0].references == [
+        Reference(output=Output(text="Even"), tags=[CORRECT_TAG]),
+        Reference(output=Output(text="Odd"), tags=[]),
+    ]
+    assert instances[0].split == "train"

helm/benchmark/scenarios/thai_exam_scenario.py

@@ -0,0 +1,135 @@
+import os
+from typing import Dict, List
+import json
+
+from helm.common.general import ensure_file_downloaded
+from helm.common.hierarchical_logger import hlog
+from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+
+
+class ThaiExamScenario(Scenario):
+    """
+    ThaiExam, a benchmark comprising Thai multiple-choice examinations as follows:
+
+    ∙ ONET: The Ordinary National Educational Test (ONET) is an examination for students in Thailand.
+    We select the grade-12 ONET exam, which comprises 5 subjects and each question has 5 choices.
+    These subjects are Thai, English, Mathematics, Social Studies, and Science.
+    Amounting to a total of 170 questions and options.
+
+    ∙ IC: The Investment Consultant (IC) examination, a licensing test for investment professionals in Thailand.
+    Developed by the Stock Exchange of Thailand (SET), features 4 choices per question.
+    We extracted questions for levels 1, 2, and 3 resulting in a total of 95 questions and options.
+
+    ∙ TGAT: The Thai General Aptitude Test (TGAT), a national high school examination in Thailand.
+    Focuses on critical and logical thinking skills.
+    We collected a total of 90 questions and answers. The TGAT consists of four choices per question.
+
+    ∙ TPAT-1: The Thai Professional Aptitude Test 1 (TPAT-1) is a national high school examination in Thailand.
+    The Exam assesses students’ professional skills requirement in medical schools.
+    This subset contains reasoning and medical ethics. We collected a total of 116 questions and answers.
+    The TPAT-1 consists of 5 choices per question.
+
+    ∙ A-Level: An academic knowledge assessment examination (Applied Knowledge Level)
+    that covers general foundational subjects taught in schools.
+    The content assessed in this examination aligns with the curriculum guidelines
+    and emphasizes the practical application of knowledge in daily life.
+    We collected a total of 175 questions and answers.
+
+    We created and used these exams to evaluate the performance of the Typhoon models(https://arxiv.org/abs/2312.13951).
+
+    Prompt models using the following format
+
+        <input>                  # train
+        A. <reference>
+        B. <reference>
+        C. <reference>
+        D. <reference>
+        E. <reference>
+        Answer: <A/B/C/D/E>
+
+        x N (N-shot)
+
+        <input>                  # test
+        A. <reference1>
+        B. <reference2>
+        C. <reference3>
+        D. <reference4>
+        E. <reference5>
+        Answer:
+
+    For example:
+
+        ในระบบย่อยอาหารของมนุษย์ การดูดซึมสารอาหารส่วนใหญ่เกิดขึ้นที่อวัยวะใด?
+        A. ลำไส้เล็ก
+        B. ตับอ่อน
+        C. ลำไส้ใหญ่
+        D. กระเพาะอาหาร
+        E. หัวใจ
+        Answer: A
+
+        ข้อใดอธิบายเกี่ยวกับแรงไฟฟ้าได้ถูกต้อง?
+        A. เกิดได้โดยที่วัตถุไม่ต้องสัมผัสกัน
+        B. เป็นได้เฉพาะแรงผลักเท่านั้น
+        C. เป็นได้เฉพาะแรงดูดเท่านั้น
+        D. เป็นแรงต้านระหว่างวัตถุเท่านั้น
+        E. ถูกทุกข้อ
+        Answer:
+
+    Target: A
+    """
+
+    name = "thai_exam"
+    description = "ThaiExam benchmark comprising Thai multiple-choice examinations."
+    tags = ["knowledge", "multiple_choice"]
+
+    def __init__(self, exam: str):
+        super().__init__()
+        self.exam = exam
+
+    def download_thai_exam(self, path: str):
+        ensure_file_downloaded(
+            "https://storage.googleapis.com/thai_dataset/thai_exam.tar.gz",
+            target_path=path,
+            unpack=True,
+        )
+
+    def process_jsonl(self, jsonl_path: str, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        hlog(f"Reading {jsonl_path}")
+        with open(jsonl_path, "r") as f:
+            for line in f:
+                data = json.loads(line)
+                # for handle missing key incase of some subject doesn't have all 5 choices
+                answers = [data[key] for key in ["a", "b", "c", "d", "e"] if key in data and data[key] != ""]
+                answers_dict = dict(zip(["A", "B", "C", "D", "E"], answers))
+
+                question, correct_answer = data["question"], answers_dict[data["answer"].upper()]
+
+                def answer_to_reference(answer: str) -> Reference:
+                    return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+                instance = Instance(
+                    input=Input(text=question),
+                    references=list(map(answer_to_reference, answers)),
+                    split=split,
+                )
+                instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "data")
+        self.download_thai_exam(data_path)
+
+        instances: List[Instance] = []
+        splits: Dict[str, str] = {
+            "train": TRAIN_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for split in splits:
+            jsonl_path: str = os.path.join(data_path, self.exam, f"{self.exam}_{split}.jsonl")
+            if not os.path.exists(jsonl_path):
+                hlog(f"{jsonl_path} doesn't exist, skipping")
+                continue
+            instances.extend(self.process_jsonl(jsonl_path, splits[split]))
+
+        return instances

helm/benchmark/scenarios/the_pile_scenario.py

@@ -50,14 +50,13 @@ class ThePileScenario(Scenario):
         self.subset = subset
 
     @htrack(None)
-    def load_and_cache_all_subsets(self, output_path):
-        data_path = os.path.join(output_path, "data")
+    def load_and_cache_all_subsets(self, data_jsonl, output_path):
         subsets: Dict[str, List] = {subset: [] for subset in self.pile_subsets}
 
         # Load all data into memory
         with htrack_block("Loading"):
-            hlog(f"Loading all data from {data_path}")
-            with open(data_path) as f:
+            hlog(f"Loading all data from {data_jsonl}")
+            with open(data_jsonl) as f:
                 data = [json.loads(line) for line in f]
 
         # Classify the documents by subset
@@ -76,10 +75,10 @@ class ThePileScenario(Scenario):
 
     def get_instances(self, output_path: str) -> List[Instance]:
         # Download the raw data
-        data_path = os.path.join(output_path, "data")
+        data_jsonl = os.path.join(output_path, "data")
         ensure_file_downloaded(
             source_url="https://the-eye.eu/public/AI/pile/test.jsonl.zst",
-            target_path=data_path,
+            target_path=data_jsonl,
             unpack=True,
         )
 
@@ -87,7 +86,7 @@ class ThePileScenario(Scenario):
 
         # If the target subset does not exist, load and cache all subsets to the directory
         if not os.path.exists(subset_path):
-            self.load_and_cache_all_subsets(output_path)
+            self.load_and_cache_all_subsets(data_jsonl, output_path)
 
         # Read all the instances
         instances = []

helm/benchmark/scenarios/unitxt_scenario.py

@@ -0,0 +1,56 @@
+from typing import List
+from datasets import load_dataset
+
+from helm.benchmark.scenarios.scenario import (
+    Output,
+    Reference,
+    Scenario,
+    Instance,
+    Input,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    VALID_SPLIT,
+)
+
+
+class UnitxtScenario(Scenario):
+    """Integration with Unitxt: https://unitxt.rtfd.io/"""
+
+    name = "unitxt"
+    description = "Unitxt Scenarios"
+    tags = ["unitxt"]
+
+    UNITXT_SPLIT_NAME_TO_HELM_SPLIT_NAME = {
+        "train": TRAIN_SPLIT,
+        "test": TEST_SPLIT,
+        "validation": VALID_SPLIT,
+    }
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        self.kwargs = kwargs
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset_name = ",".join(f"{key}={value}" for key, value in self.kwargs.items())
+        dataset = load_dataset("unitxt/data", dataset_name, trust_remote_code=True)
+
+        instances: List[Instance] = []
+
+        for unitxt_split_name, helm_split_name in UnitxtScenario.UNITXT_SPLIT_NAME_TO_HELM_SPLIT_NAME.items():
+            for index, row in enumerate(dataset[unitxt_split_name]):
+                references = [
+                    Reference(
+                        output=Output(text=reference_text),
+                        tags=[CORRECT_TAG],
+                    )
+                    for reference_text in row["references"]
+                ]
+                instance = Instance(
+                    id=f"{unitxt_split_name}{index}",
+                    input=Input(text=row["source"]),
+                    references=references,
+                    split=helm_split_name,
+                )
+                instances.append(instance)
+        return instances

helm/benchmark/scenarios/verifiability_judgment_scenario.py

@@ -137,7 +137,9 @@ class VerifiabilityJudgementScenario(Scenario):
         for _, filesplit in split_to_filesplit.items():
             target_name = f"verifiability_judgments_{filesplit}.jsonl"
             target_path: str = os.path.join(data_path, target_name)
-            url: str = f"https://github.com/nelson-liu/evaluating-verifiability-in-generative-search-engines/raw/40bf37e3a4eca7d82515df2c800ec9605458d637/verifiability_judgments/{target_name}.gz"  # noqa: E501
+            url: str = (
+                f"https://github.com/nelson-liu/evaluating-verifiability-in-generative-search-engines/raw/40bf37e3a4eca7d82515df2c800ec9605458d637/verifiability_judgments/{target_name}.gz"  # noqa: E501
+            )
             ensure_file_downloaded(source_url=url, target_path=target_path)
             assert os.path.exists(target_path)
 

helm/benchmark/scenarios/vicuna_scenario.py

@@ -26,7 +26,7 @@ class VicunaScenario(Scenario):
             return self.category == "all" or raw["category"] == self.category
 
         # Download the raw data
-        source_url = "https://raw.githubusercontent.com/lm-sys/FastChat/main/fastchat/eval/table/question.jsonl"
+        source_url = "https://raw.githubusercontent.com/lm-sys/FastChat/v0.2.5/fastchat/eval/table/question.jsonl"
         data_path: str = os.path.join(output_path, "vicuna_questions.jsonl")
 
         ensure_file_downloaded(

helm/benchmark/scenarios/vision_language/bingo_scenario.py

@@ -0,0 +1,103 @@
+import os.path
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class BingoScenario(Scenario):
+    """
+    Holistic Analysis of Hallucination in GPT-4V(ision): Bias and Interference Challenges
+
+    We introduce a new benchmark, namely, the Bias and Interference Challenges in Visual Language Models (Bingo).
+    This benchmark is designed to evaluate and shed light on the two common types of hallucinations in visual
+    language models: bias and interference. Here, bias refers to the model's tendency to hallucinate certain types
+    of responses, possibly due to imbalance in its training data. Interference pertains to scenarios where the
+    judgment of GPT-4V(ision) can be disrupted due to how the text prompt is phrased or how the input image is
+    presented. The benchmark consists of open-ended question-answer pairs, and we employ open-ended generation
+    metrics for evaluation. In the experiment, we identify a notable regional bias, whereby GPT-4V(ision) is
+    better at interpreting Western images or images with English writing compared to images from other countries
+    or containing text in other languages.
+
+
+    @article{cui2023holistic,
+    title={Holistic analysis of hallucination in gpt-4v (ision): Bias and interference challenges},
+    author={Cui, Chenhang and Zhou, Yiyang and Yang, Xinyu and Wu, Shirley and Zhang, Linjun and
+    Zou, James and Yao, Huaxiu},
+    journal={arXiv preprint arXiv:2311.03287},
+    year={2023}
+    }
+
+    Paper: https://arxiv.org/abs/2311.03287
+    """
+
+    BINGO_HUGGINGFACE_DATASET_NAME: str = "PahaII/Bingo"
+
+    IMAGE_URL: str = "https://huggingface.co/datasets/PahaII/Bingo/resolve/main/images/{image_path}?download=true"
+
+    SUBJECTS: List[str] = ["T2I", "I2I", "OCR", "Factual", "Region"]
+
+    name = "bingo"
+    description = (
+        "Evaluate multimodal models on biased and inference-challenging scenarios with five subjects"
+        " ([paper](https://arxiv.org/abs/2311.03287))."
+    )
+    tags = ["vision-language"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        assert subject in self.SUBJECTS, f"Invalid subject: {subject}"
+        self._subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        images_path: str = os.path.join(output_path, "images")
+        ensure_directory_exists(images_path)
+
+        # There is only the test split in Unicorn benchmark
+        instances: List[Instance] = []
+        question_data_files = {TEST_SPLIT: f"{self._subject}.json"}
+
+        # Process the test set
+        for row in tqdm(
+            load_dataset(
+                self.BINGO_HUGGINGFACE_DATASET_NAME,
+                data_files=question_data_files,
+                split=TEST_SPLIT,
+                cache_dir=output_path,
+            )
+        ):
+            # Download the image
+            image_path: str = row["image_path"]
+            local_image_path: str = os.path.join(output_path, image_path)
+            ensure_file_downloaded(
+                source_url=self.IMAGE_URL.format(image_path=image_path),
+                target_path=local_image_path,
+                unpack=False,
+            )
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type="image/png"),
+                MediaObject(text=row["question"], content_type="text/plain"),
+            ]
+            answer: str = row["answer"]
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/hateful_memes_scenario.py

@@ -0,0 +1,92 @@
+import os.path
+from typing import List
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    ALL_SPLITS,
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class HatefulMemesScenario(Scenario):
+    """
+    The Hateful Memes Challenge and Dataset is a competition and open source dataset designed to
+    measure progress in multimodal vision-and-language classification.
+
+    In order for AI to become a more effective tool for detecting hate speech, it must be able to understand
+    content the way people do: holistically. When viewing a meme, for example, we don’t think about the words
+    and photo independently of each other; we understand the combined meaning. This is extremely challenging
+    for machines, however, because it means they can’t analyze the text and the image separately. They must
+    combine these different modalities and understand how the meaning changes when they are presented together.
+
+    @article{kiela2020hateful,
+      title={The Hateful Memes Challenge: Detecting Hate Speech in Multimodal Memes},
+      author={Kiela, Douwe and Firooz, Hamed and Mohan, Aravind and Goswami, Vedanuj and Singh,
+      Amanpreet and Ringshia, Pratik and  Testuggine, Davide},
+      journal={arXiv preprint arXiv:2005.04790},
+      year={2020}
+    }
+
+    Paper: https://arxiv.org/abs/2005.04790
+    Website: https://ai.meta.com/tools/hatefulmemes/
+    """
+
+    IMAGE_URL: str = (
+        "https://huggingface.co/datasets/neuralcatcher/hateful_memes/resolve/main/{image_path}?download=true"
+    )
+
+    # This question is from the example on the website
+    QUESTION: str = "Is this meme mean?"
+
+    name = "hateful_memes"
+    description = "Identify if a meme is hateful or not ([paper](https://arxiv.org/abs/2005.04790))."
+    tags = ["vision-language"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        ensure_directory_exists(os.path.join(output_path, "img"))
+
+        instances: List[Instance] = []
+        for split in ALL_SPLITS:
+            for row in tqdm(
+                load_dataset(
+                    "neuralcatcher/hateful_memes",
+                    split="validation" if split == VALID_SPLIT else split,
+                    cache_dir=output_path,
+                )
+            ):
+                # Download the meme
+                image_path: str = row["img"]
+                local_image_path: str = os.path.join(output_path, image_path)
+                ensure_file_downloaded(
+                    source_url=self.IMAGE_URL.format(image_path=image_path),
+                    target_path=local_image_path,
+                    unpack=False,
+                )
+                # Some examples are missing images. Skip those for now
+                if not os.path.exists(local_image_path) or os.path.getsize(local_image_path) == 0:
+                    continue
+
+                content: List[MediaObject] = [
+                    MediaObject(location=local_image_path, content_type="image/jpeg"),
+                    MediaObject(text=self.QUESTION, content_type="text/plain"),
+                ]
+                answer: str = "Yes" if row["label"] == 1 else "No"
+                instances.append(
+                    Instance(
+                        Input(multimedia_content=MultimediaObject(content)),
+                        references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                        split=split,
+                    )
+                )
+
+        return instances

helm/benchmark/scenarios/vision_language/heim_human_eval_scenario.py

@@ -0,0 +1,113 @@
+from collections import Counter
+from dataclasses import dataclass
+from typing import Dict, List, Set
+import json
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    ALL_SPLITS,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded
+
+
+@dataclass(frozen=True)
+class HEIMHumanEvalReference(Reference):
+    # The number of human annotators who gave this reference or answer.
+    num_human_answered: int = 0
+
+
+class HEIMHumanEvalScenario(Scenario):
+    """
+    In [Holistic Evaluation of Text-To-Image Models (HEIM)](https://arxiv.org/abs/2311.04287), we evaluated 26
+    state-of-the-art text-to-image models using across 12 different aspects (e.g., toxicity mitigation, unbiasedness,
+    originality, etc.). We used human annotators through AWS Mechanical Turk to evaluate the models for some of
+    these aspects (see image below).
+    This scenario contains the AI-generated images and human annotations for the following question types:
+
+    1. Alignment
+    2. Aesthetics
+    3. Clear subject
+    4. Originality
+    5. Photorealism
+
+    Citations:
+    - HEIM: https://arxiv.org/abs/2311.04287
+    - MS COCO: https://arxiv.org/abs/1405.0312
+    """
+
+    DATASET_DOWNLOAD_URL: str = (
+        "https://worksheets.codalab.org/rest/bundles/0x502d646c366c4f1d8c4a2ccf163b958f/contents/blob/"
+    )
+    VALID_QUESTION_TYPES: Set[str] = {"alignment", "aesthetics", "clear_subject", "originality", "photorealism"}
+
+    name = "heim_human_eval"
+    description = (
+        "Images generated by text-to-image models and human annotations for HEIM "
+        "([paper](https://arxiv.org/abs/2311.04287))."
+    )
+    tags = ["vision-language", "visual question answering", "image evaluation"]
+
+    def __init__(self, question_type: str):
+        super().__init__()
+        assert question_type in self.VALID_QUESTION_TYPES, f"Invalid question type: {question_type}"
+        self._question_type: str = question_type
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the dataset
+        output_path = os.path.join(output_path, "dataset")
+        ensure_file_downloaded(
+            source_url=self.DATASET_DOWNLOAD_URL, target_path=output_path, unpack=True, unpack_type="untar"
+        )
+
+        # Load the multiple-choice questions
+        with open(os.path.join(output_path, "questions.json")) as questions_file:
+            question_info: Dict = json.load(questions_file)[self._question_type]
+
+        instances: List[Instance] = []
+        for split in ALL_SPLITS:
+            annotations_split_path: str = os.path.join(output_path, f"{self._question_type}_{split}.jsonl")
+            with open(annotations_split_path) as f:
+                # Read each line/example as a JSON object
+                for line in f.readlines():
+                    image_annotation: Dict = json.loads(line)
+                    image_path: str = os.path.join(output_path, image_annotation["image_path"])
+                    assert os.path.exists(image_path), f"Image {image_path} does not exist"
+
+                    # Get the most common human answer(s) for the question
+                    human_answers: List[str] = [str(answer) for answer in image_annotation["human_annotations"]]
+                    human_answers_to_counts = Counter(human_answers)
+                    max_count: int = max(human_answers_to_counts.values())
+                    modes: List[str] = [value for value, count in human_answers_to_counts.items() if count == max_count]
+
+                    content: List[MediaObject] = [MediaObject(location=image_path, content_type="image/png")]
+                    if "prompt" in image_annotation:
+                        # Include the prompt in the content if it exists
+                        prompt: str = image_annotation["prompt"]
+                        content.append(MediaObject(text=f"Description: {prompt}", content_type="text/plain"))
+                    content.append(MediaObject(text=question_info["question"], content_type="text/plain"))
+
+                    references: List[Reference] = [
+                        HEIMHumanEvalReference(
+                            Output(text=answer),
+                            # The mode is the most common human answer and the reference we mark as correct
+                            tags=[CORRECT_TAG] if value in modes else [],
+                            num_human_answered=human_answers_to_counts[value],
+                        )
+                        for value, answer in question_info["choices"].items()
+                    ]
+                    instances.append(
+                        Instance(
+                            Input(multimedia_content=MultimediaObject(content)),
+                            references=references,
+                            split=split,
+                        )
+                    )
+
+        return instances

helm/benchmark/scenarios/vision_language/image2structure/chart2csv_scenario.py

@@ -0,0 +1,55 @@
+from typing import List
+import os
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+
+
+class Chart2CSVScenario(Scenario):
+    """
+    Chart to CSV
+    """
+
+    name = "chart2csv"
+    description = "Convert a chart to CSV."
+    tags = ["vision-language", "image2structure"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        assert os.path.exists(output_path), f"Dataset does not exist at {output_path}"
+        instances: List[Instance] = []
+
+        charts_path: str = os.path.join(output_path, "charts")
+        ground_truths_path: str = os.path.join(output_path, "groundtruth")
+
+        for chart_file in os.listdir(charts_path):
+            if not chart_file.endswith(".png"):
+                continue
+
+            chart_path: str = os.path.join(charts_path, chart_file)
+            ground_truth_file: str = chart_file.replace(".png", ".psv")
+            ground_truth_path: str = os.path.join(ground_truths_path, ground_truth_file)
+            assert os.path.exists(ground_truth_path), f"Ground truth does not exist at {ground_truth_path}"
+
+            content: List[MediaObject] = [
+                MediaObject(location=chart_path, content_type="image/png"),
+            ]
+            with open(ground_truth_path, "r") as file:
+                ground_truth: str = file.read().replace("|", ",")
+
+            instances.append(
+                Instance(
+                    Input(multimedia_content=MultimediaObject(content)),
+                    references=[Reference(Output(text=ground_truth), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances