crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/scenarios/test_mmlu_pro_scenario.py

@@ -0,0 +1,53 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.mmlu_pro_scenario import MMLUProScenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_mmlu_pro_scenario():
+    with TemporaryDirectory() as tmpdir:
+        # Test for the "abstract_algebra" subject
+        scenario = MMLUProScenario(subject="math")
+        instances = scenario.get_instances(tmpdir)
+        assert instances[1].input == Input(
+            text="Let V be the set of all real polynomials p(x). Let transformations T, S be defined on V by T:p(x) -> xp(x) and S:p(x) -> p'(x) = d/dx p(x), and interpret (ST)(p(x)) as S(T(p(x))). Which of the following is true?"  # noqa: E501
+        )
+
+        # Ensure it handles up to 10 answer options
+        assert instances[1].references == [
+            Reference(output=Output(text="ST + TS is the identity map of V onto itself."), tags=[]),
+            Reference(output=Output(text="TS = 0"), tags=[]),
+            Reference(output=Output(text="ST = 1"), tags=[]),
+            Reference(output=Output(text="ST - TS = 0"), tags=[]),
+            Reference(output=Output(text="ST = T"), tags=[]),
+            Reference(output=Output(text="ST = 0"), tags=[]),
+            Reference(output=Output(text="ST = TS"), tags=[]),
+            Reference(output=Output(text="ST - TS is the identity map of V onto itself."), tags=[CORRECT_TAG]),
+            Reference(output=Output(text="TS = T"), tags=[]),
+            Reference(output=Output(text="ST = S"), tags=[]),
+        ]
+        assert instances[1].split == "train"
+
+        scenario = MMLUProScenario(subject="health")
+        instances = scenario.get_instances(tmpdir)
+
+        assert instances[0].input == Input(
+            text="Which of the following is the body cavity that contains the pituitary gland?"
+        )
+
+        # Check references with more answer choices and correct tagging
+        assert instances[0].references == [
+            Reference(output=Output(text="Ventral"), tags=[]),
+            Reference(output=Output(text="Dorsal"), tags=[]),
+            Reference(output=Output(text="Buccal"), tags=[]),
+            Reference(output=Output(text="Thoracic"), tags=[]),
+            Reference(output=Output(text="Pericardial"), tags=[]),
+            Reference(output=Output(text="Abdominal"), tags=[]),
+            Reference(output=Output(text="Spinal"), tags=[]),
+            Reference(output=Output(text="Pelvic"), tags=[]),
+            Reference(output=Output(text="Pleural"), tags=[]),
+            Reference(output=Output(text="Cranial"), tags=[CORRECT_TAG]),
+        ]
+        assert instances[0].split == "train"

helm/benchmark/scenarios/test_oab_exams_scenario.py

@@ -0,0 +1,51 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.oab_exams_scenario import OABExamsScenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_oab_exams_scenario():
+    oab_scenario = OABExamsScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = oab_scenario.get_instances(tmpdir)
+    assert len(instances) == 2210
+    assert instances[100].split == TEST_SPLIT
+
+    assert instances[100].input.text.startswith(
+        "O Congresso Nacional e suas respectivas Casas se reúnem anualmente para a atividade legislativa."
+    )
+    assert len(instances[100].input.text) == 178
+
+    assert instances[100].references == [
+        Reference(
+            output=Output(
+                text="Legislatura: o período compreendido entre 2 de fevereiro a 17 de julho e 1º de agosto a 22 de dezembro."  # noqa: E501
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="Sessão legislativa: os quatro anos equivalentes ao mandato dos parlamentares."  # noqa: E501
+            ),
+            tags=[],
+        ),
+        Reference(
+            output=Output(
+                text="Sessão conjunta: a reunião da Câmara dos Deputados e do Senado Federal destinada, por exemplo, "
+                "a conhecer do veto presidencial e sobre ele deliberar."  # noqa: E501
+            ),
+            tags=[CORRECT_TAG],
+        ),
+        Reference(
+            output=Output(
+                text="Sessão extraordinária: a que ocorre por convocação ou do Presidente do Senado Federal ou do "
+                "Presidente da Câmara dos Deputados ou do Presidente da República e mesmo por requerimento da maioria "
+                "dos membros de ambas as Casas para, excepcionalmente, inaugurar a "
+                "sessão legislativa e eleger as respectivas mesas diretoras."  # noqa: E501
+            ),
+            tags=[],
+        ),
+    ]
+    assert instances[100].references[2].is_correct

helm/benchmark/scenarios/test_omni_math_scenario.py

@@ -0,0 +1,27 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.omni_math_scenario import OmniMATHScenario
+from helm.benchmark.scenarios.scenario import Input, TEST_SPLIT
+
+
+@pytest.mark.scenarios
+def test_omni_math_scenario_get_instances():
+    omni_math_scenario = OmniMATHScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = omni_math_scenario.get_instances(tmpdir)
+    assert len(instances) == 4428
+    assert instances[0].input == Input(
+        text=(
+            "Let $ n(\\ge2) $ be a positive integer. Find the minimum $ m $, "
+            "so that there exists $x_{ij}(1\\le i ,j\\le n)$ satisfying:\n(1)For every "
+            "$1\\le i ,j\\le n, x_{ij}=max\\{x_{i1},x_{i2},...,x_{ij}\\} $ or $ x_{ij}="
+            "max\\{x_{1j},x_{2j},...,x_{ij}\\}.$\n(2)For every $1\\le i \\le n$, there "
+            "are at most $m$ indices $k$ with $x_{ik}=max\\{x_{i1},x_{i2},...,x_{ik}\\}."
+            "$\n(3)For every $1\\le j \\le n$, there are at most $m$ indices $k$ with "
+            "$x_{kj}=max\\{x_{1j},x_{2j},...,x_{kj}\\}.$"
+        )
+    )
+    assert instances[0].split == TEST_SPLIT
+    assert instances[0].references
+    assert instances[0].references[0].output.text == "1 + \\left\\lceil \\frac{n}{2} \\right\\rceil"

helm/benchmark/scenarios/test_tweetsentbr_scenario.py

@@ -0,0 +1,24 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.tweetsentbr_scenario import TweetSentBRScenario
+from helm.benchmark.scenarios.scenario import TRAIN_SPLIT, CORRECT_TAG, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_tweetsentbr_scenario():
+    tweetsentbr = TweetSentBRScenario()
+    with TemporaryDirectory() as tmpdir:
+        instances = tweetsentbr.get_instances(tmpdir)
+    assert len(instances) == 2085
+    assert instances[0].split == TRAIN_SPLIT
+
+    assert instances[0].input.text.startswith("joca tá com a corda toda 😂 😂 😂 😂")
+    assert len(instances[0].input.text) == 32
+
+    assert instances[0].references == [
+        Reference(
+            output=Output(text="Positivo"),
+            tags=[CORRECT_TAG],
+        )
+    ]

helm/benchmark/scenarios/test_wildbench_scenario.py

@@ -0,0 +1,15 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.wildbench_scenario import WildBenchScenario
+from helm.benchmark.scenarios.scenario import TEST_SPLIT
+
+
+@pytest.mark.scenarios
+def test_wildbench_scenario_get_instances():
+    wildbench_scenario = WildBenchScenario(subset="v2")
+    with TemporaryDirectory() as tmpdir:
+        instances = wildbench_scenario.get_instances(tmpdir)
+    assert len(instances) == 1024
+    assert instances[0].split == TEST_SPLIT
+    assert instances[0].extra_data

helm/benchmark/scenarios/test_winogrande_afr_scenario.py

@@ -0,0 +1,19 @@
+import pytest
+from tempfile import TemporaryDirectory
+
+from helm.benchmark.scenarios.winogrande_afr_scenario import Winogrande_Afr_Scenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Input, Output, Reference
+
+
+@pytest.mark.scenarios
+def test_winogrande_afr_scenario():
+    with TemporaryDirectory() as tmpdir:
+        scenario = Winogrande_Afr_Scenario(lang="am")
+        instances = scenario.get_instances(tmpdir)
+        assert len(instances) == 3674
+        assert instances[0].input == Input(text="ሳራ ከማሪያ በጣም የተሻለች የቀዶ ጥገና ሐኪም ስለነበረች ሁልጊዜ _ ቀላል ህመሞችን ታክማለች.")
+        assert instances[0].references == [
+            Reference(output=Output(text="ሳራ"), tags=[]),
+            Reference(output=Output(text="ማሪያ"), tags=[CORRECT_TAG]),
+        ]
+        assert instances[0].split == "train"

helm/benchmark/scenarios/thai_exam_scenario.py

@@ -4,7 +4,16 @@ import json
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 class ThaiExamScenario(Scenario):

helm/benchmark/scenarios/the_pile_scenario.py

@@ -7,7 +7,7 @@ from typing import Dict, List
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog, htrack, htrack_block
-from .scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
 
 
 class ThePileScenario(Scenario):

helm/benchmark/scenarios/truthful_qa_scenario.py

@@ -3,7 +3,16 @@ import os
 from typing import List, Dict, Any
 
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
-from .scenario import Scenario, Instance, Reference, VALID_SPLIT, TRAIN_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    VALID_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 class TruthfulQAScenario(Scenario):

helm/benchmark/scenarios/tweetsentbr_scenario.py

@@ -0,0 +1,66 @@
+from typing import Any, List, Dict
+from pathlib import Path
+from datasets import load_dataset
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class TweetSentBRScenario(Scenario):
+    """
+    TweetSentBR is a corpus of Tweets in Brazilian Portuguese. It was labeled by several
+    annotators following steps stablished on the literature for improving reliability on
+    the task of Sentiment Analysis. Each Tweet was annotated in one of the three following classes:
+
+    Positive - tweets where a user meant a positive reaction or evaluation about the main topic on the post;
+    Negative - tweets where a user meant a negative reaction or evaluation about the main topic on the post;
+    Neutral - tweets not belonging to any of the last classes, usually not making a point, out of topic,
+    irrelevant, confusing or containing only objective data.
+
+    This dataset is a subset of the tweetSentBR, it contains only 75 samples from the training set
+    and all 2.000+ instances of the test set. This is meant for evaluating language models in a few-shot setting.
+    """
+
+    name = "simple_classification"
+    description = "Classify tweets into Positive, Negative or Neutral."
+    tags = ["classification"]
+
+    def process_dataset(self, dataset: Any, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        label_names = {"Positive": "Positivo", "Negative": "Negativo", "Neutral": "Neutro"}
+        for example in dataset[split]:
+            input = Input(text=example["sentence"])
+            # NOTE: For classification scenarios, the reference outputs should be the same
+            # for all instances, and should include both correct and incorrect classes.
+            # HELM only supports single-label classification. Exactly one reference
+            # should have the CORRECT_TAG tag.
+            references = [
+                Reference(Output(text=label_names[example["label"]]), tags=[CORRECT_TAG]),
+            ]
+            instance = Instance(input=input, references=references, split=split)
+            instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+        dataset = load_dataset("eduagarcia/tweetsentbr_fewshot", cache_dir=cache_dir)
+        splits: Dict[str, str] = {
+            "train": TRAIN_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for split in splits:
+            if split not in splits.keys():
+                hlog(f"{split} split doesn't exist, skipping")
+                continue
+            instances.extend(self.process_dataset(dataset, splits[split]))
+
+        return instances

helm/benchmark/scenarios/twitter_aae_scenario.py

@@ -4,7 +4,7 @@ from typing import List
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
 
 CODALAB_URI_TEMPLATE: str = (
     "https://worksheets.codalab.org/rest/bundles/0x31485f8c37ad481fb9f4e9bf7ccff6e5/contents/blob/"

helm/benchmark/scenarios/unitxt_scenario.py

@@ -32,13 +32,19 @@ class UnitxtScenario(Scenario):
         self.kwargs = kwargs
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        dataset_name = ",".join(f"{key}={value}" for key, value in self.kwargs.items())
+        if len(self.kwargs) == 1 and "recipe" in self.kwargs:
+            dataset_name = self.kwargs["recipe"]
+        else:
+            dataset_name = ",".join(f"{key}={value}" for key, value in self.kwargs.items())
         dataset = load_dataset("unitxt/data", dataset_name, trust_remote_code=True)
 
         instances: List[Instance] = []
 
         for unitxt_split_name, helm_split_name in UnitxtScenario.UNITXT_SPLIT_NAME_TO_HELM_SPLIT_NAME.items():
-            for index, row in enumerate(dataset[unitxt_split_name]):
+            dataset_split = dataset.get(unitxt_split_name)
+            if dataset_split is None:
+                continue
+            for index, row in enumerate(dataset_split):
                 references = [
                     Reference(
                         output=Output(text=reference_text),

helm/benchmark/scenarios/verifiability_judgment_scenario.py

@@ -4,7 +4,7 @@ import json
 from typing import Dict, List, Tuple
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,

helm/benchmark/scenarios/vicuna_scenario.py

@@ -3,7 +3,7 @@ import os
 from typing import List
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
 
 
 class VicunaScenario(Scenario):

helm/benchmark/scenarios/vision_language/blink_scenario.py

@@ -0,0 +1,140 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.images_utils import generate_hash
+
+
+class BlinkScenario(Scenario):
+    """
+    BLINK is a benchmark containing 14 visual perception tasks that can be solved by humans “within a blink”,
+    but pose significant challenges for VLMs.
+
+    Website: https://zeyofu.github.io/blink/
+
+    @article{fu2024blink,
+        title={BLINK: Multimodal Large Language Models Can See but Not Perceive},
+        author={Fu, Xingyu and Hu, Yushi and Li, Bangzheng and Feng, Yu and Wang, Haoyu and Lin, Xudong and Roth,
+        Dan and Smith, Noah A and Ma, Wei-Chiu and Krishna, Ranjay},
+        journal={arXiv preprint arXiv:2404.12390},
+        year={2024}
+    }
+    """
+
+    HUGGINGFACE_DATASET_NAME: str = "BLINK-Benchmark/BLINK"
+
+    VALID_CATEGORIES: List[str] = [
+        "Art_Style",
+        "Counting",
+        "Forensic_Detection",
+        "Functional_Correspondence",
+        "IQ_Test",
+        "Jigsaw",
+        "Multi-view_Reasoning",
+        "Object_Localization",
+        "Relative_Depth",
+        "Relative_Reflectance",
+        "Semantic_Correspondence",
+        "Spatial_Relation",
+        "Visual_Correspondence",
+        "Visual_Similarity",
+    ]
+
+    name = "blink"
+    description = (
+        "BLINK is a benchmark containing 14 visual perception tasks that can be solved by humans within a blink, "
+        "but pose significant challenges for VLMs. ([Fu, 2024](https://arxiv.org/abs/2404.12390))."
+    )
+    tags = ["vision-language", "knowledge", "reasoning"]
+
+    def __init__(self, category: str):
+        super().__init__()
+
+        if category not in self.VALID_CATEGORIES:
+            raise ValueError(f"Invalid category: {category}. Valid categories are: {self.VALID_CATEGORIES}")
+        self._category: str = category
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        def save_image(image) -> str:
+            image_file_name: str = generate_hash(image) + ".jpg"
+            local_image_path: str = os.path.join(output_path, image_file_name)
+            if not os.path.exists(local_image_path):
+                image.save(local_image_path)
+            return local_image_path
+
+        def get_image_header(image_index: int) -> str:
+            if image_index == 1:
+                return "First image:"
+            elif image_index == 2:
+                return "Second image:"
+            elif image_index == 3:
+                return "Third image:"
+            elif image_index == 4:
+                return "Fourth image:"
+            else:
+                raise ValueError(f"Invalid image index: {image_index}")
+
+        instances: List[Instance] = []
+        for row in tqdm(
+            load_dataset(self.HUGGINGFACE_DATASET_NAME, self._category, split="val", cache_dir=output_path)
+        ):
+            # Save the image(s) to disk
+            has_multiple_images: bool = row["image_2"] is not None
+            content: List[MediaObject] = []
+
+            if has_multiple_images:
+                # An example can have up to 4 images
+                for i in range(1, 5):
+                    image_i = row[f"image_{i}"]
+                    if image_i is None:
+                        break
+
+                    # Before each image, include a header text that indicates which number image it is.
+                    # Some prompts refer to specific image numbers within the question, e.g.,
+                    # "Given three similar but different images, take the first image as reference.
+                    # Can you tell which one of the latter two images is most similar to the first one?
+                    # Select from the following choices. (A) the second image (B) the third image"
+                    image_path: str = save_image(image_i)
+                    content.extend(
+                        [
+                            MediaObject(text=get_image_header(i), content_type="text/plain"),
+                            MediaObject(location=image_path, content_type="image/jpeg"),
+                        ]
+                    )
+            else:
+                image1 = row["image_1"]
+                image1_path: str = save_image(image1)
+                content.append(MediaObject(location=image1_path, content_type="image/jpeg"))
+
+            # Add the prompt that has both the question and the answer choices
+            prompt: str = row["prompt"]
+            # Replace (A), (B), (C), (D) with \nA. \nB. \nC. \nD. since we are just expecting the letter answer
+            prompt = prompt.replace("(A)", "\nA.").replace("(B)", "\nB.").replace("(C)", "\nC.").replace("(D)", "\nD.")
+            content.append(MediaObject(text=prompt, content_type="text/plain"))
+
+            # The answer has the correct letter choices surrounded by parentheses
+            paren_letter_answer: str = row["answer"]
+            assert (
+                paren_letter_answer[0] == "(" and paren_letter_answer[-1] == ")"
+            ), f"Unexpected answer format: {paren_letter_answer}"
+            letter_answer: str = paren_letter_answer[1]
+            references: List[Reference] = [
+                Reference(output=Output(text=letter_answer), tags=[CORRECT_TAG]),
+            ]
+            instances.append(
+                Instance(Input(multimedia_content=MultimediaObject(content)), references=references, split=VALID_SPLIT)
+            )
+
+        return instances

helm/benchmark/scenarios/vision_language/mm_star_scenario.py

@@ -0,0 +1,95 @@
+from typing import List
+import os
+
+from datasets import load_dataset
+from tqdm import tqdm
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Instance,
+    Input,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.images_utils import generate_hash
+
+
+class MMStarScenario(Scenario):
+    """
+    MM-STAR is an elite vision-indispensable multi-modal benchmark comprising 1,500 challenge samples meticulously
+    selected by humans. MMStar is designed to benchmark 6 core capabilities and 18 detailed axes, aiming to evaluate
+    the multi-modal capacities of LVLMs with a carefully balanced and purified selection of samples. The samples
+    are first roughly selected from current benchmarks with an automated pipeline, strict human review is then
+    involved to ensure each selected sample exhibits visual dependency, minimal data leakage, and requires advanced
+    multi-modal capabilities for the solution.
+
+    Website: https://mmstar-benchmark.github.io/
+
+    @article{chen2024we,
+      title={Are We on the Right Way for Evaluating Large Vision-Language Models?},
+      author={Chen, Lin and Li, Jinsong and Dong, Xiaoyi and Zhang, Pan and Zang, Yuhang and Chen, Zehui and Duan,
+      Haodong and Wang, Jiaqi and Qiao, Yu and Lin, Dahua and others},
+      journal={arXiv preprint arXiv:2403.20330},
+      year={2024}
+    }
+    """
+
+    HUGGINGFACE_DATASET_NAME: str = "Lin-Chen/MMStar"
+
+    VALID_CATEGORIES: List[str] = [
+        "coarse perception",
+        "fine-grained perception",
+        "instance reasoning",
+        "logical reasoning",
+        "math",
+        "science technology",
+    ]
+
+    name = "mm_star"
+    description = (
+        "MM-STAR is an elite vision-indispensable multi-modal benchmark comprising 1,500 challenge samples "
+        "meticulously selected by humans."
+        "([Chen, 2024](https://arxiv.org/abs/2403.20330))."
+    )
+    tags = ["vision-language", "knowledge", "reasoning"]
+
+    def __init__(self, category: str):
+        super().__init__()
+
+        category = category.replace("_", " ")
+        if category not in self.VALID_CATEGORIES:
+            raise ValueError(f"Invalid category: {category}. Valid categories are: {self.VALID_CATEGORIES}")
+        if category == "science technology":
+            category = "science & technology"
+
+        self._category: str = category
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+
+        for row in tqdm(load_dataset(self.HUGGINGFACE_DATASET_NAME, split="val", cache_dir=output_path)):
+            # Filter by category
+            category: str = row["category"]
+            if category != self._category:
+                continue
+
+            # Save the image to disk
+            image = row["image"]
+            image_file_name: str = generate_hash(image) + ".jpg"
+            local_image_path: str = os.path.join(output_path, image_file_name)
+            if not os.path.exists(local_image_path):
+                image.save(local_image_path)
+
+            content: List[MediaObject] = [
+                MediaObject(location=local_image_path, content_type="image/jpeg"),
+                MediaObject(text=row["question"], content_type="text/plain"),
+            ]
+            references: List[Reference] = [Reference(output=Output(text=row["answer"]), tags=[CORRECT_TAG])]
+            instances.append(
+                Instance(Input(multimedia_content=MultimediaObject(content)), references=references, split=VALID_SPLIT)
+            )
+
+        return instances