crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/headqa_scenario.py

@@ -0,0 +1,158 @@
+import os
+from typing import List, Optional
+
+from datasets import DatasetDict, load_dataset
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Instance,
+    Output,
+    Reference,
+    Scenario,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class HeadQAScenario(Scenario):
+    """
+    From "HEAD-QA: A Healthcare Dataset for Complex Reasoning" (Vilares et al.), HEAD-QA is a multi-choice
+    question-answering dataset designed to evaluate reasoning on challenging healthcare-related questions.
+    The questions are sourced from Spanish healthcare exams for specialized positions, covering various topics
+    such as Medicine, Nursing, Psychology, Chemistry, Pharmacology, and Biology.
+
+    Example from the dataset:
+
+    Question:
+    The excitatory postsynaptic potentials:
+
+    A) They are all or nothing.
+    B) They are hyperpolarizing.
+    C) They can be added.
+    D) They spread long distances.
+
+    Answer:
+    The answer is C. Explanation: None provided in this dataset.
+
+    @InProceedings{HEAD-QA,
+    author = {David Vilares and Manuel Vilares and Carlos Gómez-Rodríguez},
+    title = {HEAD-QA: A Healthcare Dataset for Complex Reasoning},
+    year = {2019},
+    abstract = {We present HEAD-QA, a multi-choice question answering testbed to encourage research on complex
+    reasoning. The questions come from exams to access a specialized position in the Spanish healthcare system,
+    and are challenging even for highly specialized humans. We then consider monolingual (Spanish) and
+    cross-lingual (to English) experiments with information retrieval and neural techniques. We show that:
+    (i) HEAD-QA challenges current methods, and (ii) the results lag well behind human performance,
+    demonstrating its usefulness as a benchmark for future work.}}
+
+
+    Task:
+    Given a question and its multiple-choice answers, models must identify the correct answer, corresponding to the
+    `ra` field in the dataset. The dataset spans six healthcare domains and is challenging even for experts.
+    """
+
+    HUGGING_FACE_DATASET_PATH: str = "dvilares/head_qa"
+    SKIP_VQA: bool = True
+    SKIP_TEXTQA: bool = False
+
+    name = "head_qa"
+    description = (
+        "HeadQA is a benchmark consisting of biomedical multiple-choice questions intended to"
+        "evaluate a model's medical knowledge and reasoning. Each instance presents a clinical"
+        "or scientific question with four answer options, requiring the model to select the most"
+        "appropriate answer."
+    )
+    tags = ["question_answering", "biomedical", "medicine"]
+
+    def __init__(self, language: str = "en", category: Optional[str] = None):
+        """Initialize the HEAD-QA scenario.
+
+        Args:
+            language (str, optional): Language of the dataset. Defaults to "en".
+            category (str, optional): Category of the dataset. If None, all categories are used.
+        """
+        super().__init__()
+        self.language: str = language
+        self.category: Optional[str] = category
+        assert (
+            self.SKIP_VQA or self.SKIP_TEXTQA
+        ), "Failed to initialize HeadQAScenario, one of `SKIP_VQA` or `SKIP_TEXTQA` must be True."
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "data")
+        ensure_directory_exists(data_path)
+        dataset: DatasetDict = load_dataset(self.HUGGING_FACE_DATASET_PATH, self.language)
+
+        # XXX: Should we consider validation as test too?
+        # splits = {TRAIN_SPLIT: ["train", "validation"], TEST_SPLIT: ["test"]}
+        # Limit to zero shot setting
+        splits = {TEST_SPLIT: ["test"]}
+        instances: List[Instance] = []
+        for (
+            helm_split_name,
+            dataset_splits_name,
+        ) in splits.items():  # Iterate over the splits
+            for dataset_split_name in dataset_splits_name:
+                split_data = dataset[dataset_split_name]
+
+                for example in split_data:
+                    # Whether to process Visual Question Answering (VQA) examples
+                    if self.SKIP_VQA and example["image"] is not None:
+                        continue
+
+                    # Whether to process Text Question Answering (TextQA) examples
+                    if self.SKIP_TEXTQA and example["image"] is None:
+                        continue
+
+                    # If specified, filter by category
+                    if self.category is not None:
+                        if example["category"] != self.category:
+                            continue
+
+                    question = example["qtext"]
+
+                    # Format the final answer with explanation
+                    instances.append(
+                        Instance(
+                            input=Input(text=question),
+                            references=[
+                                Reference(
+                                    Output(text=option["atext"]),
+                                    tags=[CORRECT_TAG] if option["aid"] == example["ra"] else [],
+                                )
+                                for option in example["answers"]
+                            ],
+                            split=helm_split_name,
+                            extra_data={
+                                "id": example["qid"],
+                                "name": example["name"],
+                                "category": example["category"],
+                                "year": example["year"],
+                            },
+                        )
+                    )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="head_qa",
+            display_name="HeadQA",
+            description="HeadQA is a benchmark consisting of biomedical multiple-choice questions "
+            "intended to evaluate a model's medical knowledge and reasoning. Each instance "
+            "presents a clinical or scientific question with four answer options, requiring "
+            "the model to select the most appropriate answer [(Vilares et al., "
+            "2019)](https://arxiv.org/abs/1906.04701).",
+            taxonomy=TaxonomyInfo(
+                task="Question answering",
+                what="Medical knowledge testing",
+                when="Any",
+                who="Medical student, Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/healthqa_br_scenario.py

@@ -0,0 +1,80 @@
+from typing import Any, List
+import re
+from pathlib import Path
+from datasets import load_dataset
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class HEALTHQA_BR_Scenario(Scenario):
+    """
+    HealthQA-BR is a large-scale benchmark designed to evaluate the clinical knowledge of Large Language Models (LLMs)
+    within the Brazilian Unified Health System (SUS) context. It comprises 5,632 multiple-choice questions sourced from
+    nationwide licensing exams and residency tests, reflecting real challenges faced by Brazil's public health sector.
+    Unlike benchmarks focused on the U.S. medical landscape, HealthQA-BR targets the Brazilian healthcare ecosystem,
+    covering a wide range of medical specialties and interdisciplinary professions such as nursing, dentistry,
+    psychology, social work, pharmacy, and physiotherapy. This comprehensive approach enables a detailed assessment
+    of AI models’ ability to collaborate effectively in the team-based patient care typical of SUS.
+    """
+
+    name = "healthqa_br"
+    description = "MQA benchmark with questions from Brazilian entrance exams"
+    tags = ["knowledge", "multiple_choice", "pt-br"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data and read all the dialogues
+        dataset: Any
+        # Read all the instances
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+
+        dataset = load_dataset("Larxel/healthqa-br", cache_dir=cache_dir)
+        for example in dataset["train"]:
+            question_choices = example["question"]
+            answer = example["answer"].strip().upper()
+
+            # Separate the question statement from the alternatives
+            question_text, choices_text = self.split_question_and_choices(question_choices)
+
+            # Extract alternatives from text choices_text
+            pattern = r"'([A-Z])':\s*'([^']+)'"
+            matches = re.findall(pattern, choices_text)
+            answers_dict = {label: text for label, text in matches}
+
+            if answer not in answers_dict:
+                continue
+
+            correct_answer_text = answers_dict[answer]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if correct_answer_text == answer else [])
+
+            instance = Instance(
+                input=Input(text=question_text),
+                split=TEST_SPLIT,
+                references=[answer_to_reference(text) for text in answers_dict.values()],
+            )
+            instances.append(instance)
+        return instances
+
+    def split_question_and_choices(self, full_text: str):
+        # Search for the first occurrence of the alternative pattern
+        match = re.search(r"\n'[A-Z]':\s*'.+", full_text)
+        if match:
+            # Everything before the alternatives
+            question_part = full_text[: match.start()].strip()
+            # All of the alternatives (from match to end)
+            choices_part = full_text[match.start() :].strip()
+        else:
+            # If you don't find a pattern, consider everything as a question, and no alternative.
+            question_part = full_text.strip()
+            choices_part = ""
+
+        return question_part, choices_part

helm/benchmark/scenarios/helpdesk_call_summarization_scenario.py

@@ -0,0 +1,50 @@
+import csv
+import os
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    Input,
+    ScenarioMetadata,
+)
+
+
+_DATA_DIRRECTORY_PATH = "restricted/helpdesk_call_summarization/HELM Sample Transcripts_20241221_0045"
+
+
+class HelpdeskCallSummarizationScenario(Scenario):
+    """Helpdesk call summarization."""
+
+    name = "helpdesk_call_summarization"
+    description = "Helpdesk call summarization."
+    tags = ["helpdesk_call_center"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        for file_name in os.listdir(_DATA_DIRRECTORY_PATH):
+            if not file_name.endswith(".csv") or not file_name.startswith("Call1-"):
+                continue
+            file_path = os.path.join(_DATA_DIRRECTORY_PATH, file_name)
+            with open(file_path) as f:
+                csv_reader = csv.reader(f)
+                prompt_lines = [f"{row[0]}: {row[4]}" for row in csv_reader]
+            prompt = "\n".join(prompt_lines)
+            instance_id = file_name.removeprefix("Call1-").removesuffix(".csv")
+            input = Input(text=prompt)
+            instance = Instance(id=instance_id, input=input, references=[], split=TEST_SPLIT)
+            instances.append(instance)
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="helpdesk_call_summarization",
+            display_name="Helpdesk Call summarization",
+            short_display_name=None,
+            description="Helpdesk Call summarization",
+            taxonomy=TaxonomyInfo(task="summarization", what="n/a", when="?", who="n/a", language="English"),
+            main_metric="unknown",
+            main_split="test",
+        )

helm/benchmark/scenarios/ice_scenario.py

@@ -4,9 +4,10 @@ from typing import List, Union
 from enum import Enum
 import pandas as pd
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.optional_dependencies import handle_module_not_found_error
-from .ice_scenario_pinned_file_order import listdir_with_pinned_file_order
-from .scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.ice_scenario_pinned_file_order import listdir_with_pinned_file_order
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 try:
     # pd.read_excel() uses xlrd
@@ -114,8 +115,12 @@ class ICEScenario(Scenario):
     """
     The International Corpus of English (ICE).
 
-    NOTE: This text cannot be downloaded
-    automatically. You must extract each subset zip file into /benchmark_output/scenarios/ice.
+    NOTE: This text cannot be downloaded automatically.
+    You must extract each subset zip file into args.output_path + '/scenarios/ice',
+    which is by default '/benchmark_output/scenarios/ice',
+    where args.output_path is parsed from the command line argument.
+    See helm.benchmark.runner for more details about args.output_path.
+
     The archives should extract into folders named according to the dictionary SUBSET_TO_DIRECTORY
     below.
 
@@ -463,3 +468,22 @@ class ICEScenario(Scenario):
                     instances.append(Instance(Input(text=t), references=[], split=TEST_SPLIT))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="ice",
+            display_name="ICE (International Corpus of English)",
+            short_display_name="ICE",
+            description="The International Corpus of English (ICE) drawn from English speakers from "
+            "various places in the world, initiated by [Greenbaum "
+            "(1991)](https://www.cambridge.org/core/journals/english-today/article/abs/ice-the-international-corpus-of-english/47808205394C538393C3FD8E62E5E701).",
+            taxonomy=TaxonomyInfo(
+                task="language modeling",
+                what="?",
+                when="?",
+                who="?",
+                language="English varieties from different nations",
+            ),
+            main_metric="bits_per_byte",
+            main_split="test",
+        )

helm/benchmark/scenarios/ifeval_scenario.py

@@ -0,0 +1,71 @@
+import datasets
+import os
+from typing import List
+from helm.benchmark.presentation.schema import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    ScenarioMetadata,
+    Instance,
+    Input,
+    TEST_SPLIT,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class IFEvalScenario(Scenario):
+    """IFEval
+
+    IFEval contains around 500 "verifiable instructions" such as "write in more than 400 words"
+    and "mention the keyword of AI at least 3 times" which can be verified by heuristics."""
+
+    name = "ifeval"
+    description = "Instruction-Following Evaluation for Large Language Models"
+    tags = ["instruction following"]
+
+    def __init__(self):
+        super().__init__()
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get IFEval from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "google/IFEval",
+            trust_remote_code=True,
+            cache_dir=cache_dir,
+            split="train",
+            revision="966cd89545d6b6acfd7638bc708b98261ca58e84",
+        )
+        assert isinstance(dataset, datasets.Dataset)
+
+        # Read all instances
+        instances: List[Instance] = []
+        for _, row in enumerate(dataset):
+            id = row["key"]
+            input = Input(text=row["prompt"].strip())
+            instance = Instance(
+                id=f"id{id}",
+                input=input,
+                references=[],
+                split=TEST_SPLIT,
+                extra_data={"instruction_ids": row["instruction_id_list"], "instruction_kwargs": row["kwargs"]},
+            )
+            instances.append(instance)
+
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="IFEval",
+            description=self.description,
+            main_metric="ifeval_strict_accuracy",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="instruction following",
+                what="verifiable general domain instruction following",
+                who="human annotators",
+                when="2023",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/image_generation/common_syntactic_processes_scenario.py

@@ -0,0 +1,105 @@
+from typing import List, Dict
+
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+
+
+class CommonSyntacticProcessesScenario(Scenario):
+    """
+    From "DALL-E 2 Fails to Reliably Capture Common Syntactic Processes", DALL-E performs poorly
+    when given prompts from 8 different grammatical phenomena:
+
+    1. Binding principles and coreference
+    2. Passives
+    3. Word order
+    4. Coordination
+    5. Comparatives
+    6. Negation
+    7. Ellipsis
+    8. Structural ambiguity
+
+    The benchmark has 5 examples per grammatical phenomenon (see the full list below), where
+    each example can have multiple prompts. The authors generated 4 images per prompt.
+
+    Paper: https://arxiv.org/abs/2210.12889
+    """
+
+    BINDING_PRINCIPLES: str = "binding_principles"
+    PASSIVES: str = "passives"
+    WORD_ORDER: str = "word_order"
+    COORDINATION: str = "coordination"
+    COMPARATIVES: str = "comparatives"
+    NEGATION: str = "negation"
+    ELLIPSIS: str = "ellipsis"
+    STRUCTURAL_AMBIGUITY: str = "ambiguity"
+
+    # All prompts and example outputs are available in Table 1 of the appendix
+    PROMPT_TO_PHENOMENON: Dict[str, str] = {
+        "The man paints a picture of him": BINDING_PRINCIPLES,  # 1
+        "The man paints a picture of himself": BINDING_PRINCIPLES,  # 1
+        "The woman paints a portrait of her": BINDING_PRINCIPLES,  # 2
+        "The woman paints a portrait of herself": BINDING_PRINCIPLES,  # 2
+        "The boy looks at a picture of him": BINDING_PRINCIPLES,  # 3
+        "The boy looks at a picture of himself": BINDING_PRINCIPLES,  # 3
+        "The young lady looks at a picture of her": BINDING_PRINCIPLES,  # 4
+        "The young lady looks at a picture of herself": BINDING_PRINCIPLES,  # 4
+        "The man takes a picture of him": BINDING_PRINCIPLES,  # 5
+        "The man takes a picture of himself": BINDING_PRINCIPLES,  # 5
+        "The woman broke the vase": PASSIVES,  # 6
+        "The vase was broken by the woman": PASSIVES,  # 6
+        "The plate was broken by the woman": PASSIVES,  # 7
+        "The glass was broken by the man": PASSIVES,  # 8
+        "The jar was broken by the man": PASSIVES,  # 9
+        "The flowerpot was broken by the man": PASSIVES,  # 10
+        "The dog is chasing the man": WORD_ORDER,  # 11
+        "The man is chasing the dog": WORD_ORDER,  # 11
+        "The man gave the letter to the woman": WORD_ORDER,  # 12
+        "The man gave the woman the letter": WORD_ORDER,  # 12
+        "The man is watering the plant": WORD_ORDER,  # 13
+        "The plant is watering the man": WORD_ORDER,  # 13
+        "The mother combs the boy": WORD_ORDER,  # 14
+        "The boy combs the mother": WORD_ORDER,  # 14
+        "The man gave the comb to the woman": WORD_ORDER,  # 15
+        "The man gave the woman the comb": WORD_ORDER,  # 15
+        "The man is drinking water and the woman is drinking orange juice": COORDINATION,  # 16
+        "The woman is eating red apple and the man is eating a green apple": COORDINATION,  # 17
+        "The cat is wearing two red socks and the dog is wearing one red sock": COORDINATION,  # 18
+        "The boy wears a red hat and the girl wears a blue tie": COORDINATION,  # 19
+        "The woman is washing the dishes and the man is washing the floor": COORDINATION,  # 20
+        "The bowl has more cucumbers than strawberries": COMPARATIVES,  # 21
+        "The bowl has fewer strawberries than cucumbers": COMPARATIVES,  # 22
+        "The plate has more peas than carrots": COMPARATIVES,  # 23
+        "The plate has fewer carrots than peas": COMPARATIVES,  # 24
+        "The plate has more than seven eggs": COMPARATIVES,  # 25
+        "A tall woman without a handbag": NEGATION,  # 26
+        "A man with a red sweater and blue sweater and he is not wearing the former": NEGATION,  # 27
+        "A rainy street without cars": NEGATION,  # 28
+        "A boy with a green t-shirt without red buttons": NEGATION,  # 29
+        "A tall tree not green or black": NEGATION,  # 30
+        "The man is eating a sandwich and the woman an apple": ELLIPSIS,  # 31
+        "The man eats pizza but the woman does not": ELLIPSIS,  # 32
+        "The girl starts a sandwich and the boy a book": ELLIPSIS,  # 33
+        "The man drinks water and the woman orange juice": ELLIPSIS,  # 34
+        "The woman wears a blue shirt, but the man does not": ELLIPSIS,  # 35
+        "The man saw the boy in his car": STRUCTURAL_AMBIGUITY,  # 36
+        "The man saw the lion with the binoculars": STRUCTURAL_AMBIGUITY,  # 37
+        "The boy saw the girl using a magnifying glass": STRUCTURAL_AMBIGUITY,  # 38
+        "There are three boys and each is wearing a hat": STRUCTURAL_AMBIGUITY,  # 39
+        "Two cars painted a different color": STRUCTURAL_AMBIGUITY,  # 40
+        "Two cars each painted a different color": STRUCTURAL_AMBIGUITY,  # 40
+    }
+
+    name = "common_syntactic_processes"
+    description = "Prompts from 8 different grammatical phenomena ([paper](https://arxiv.org/abs/2210.12889))."
+    tags = ["text-to-image"]
+
+    def __init__(self, phenomenon: str):
+        super().__init__()
+        self.phenomenon: str = phenomenon
+
+    def get_instances(self, _) -> List[Instance]:
+        return [
+            # There are no reference images
+            Instance(Input(text=prompt), references=[], split=TEST_SPLIT)
+            for prompt, phenomenon in self.PROMPT_TO_PHENOMENON.items()
+            if phenomenon == self.phenomenon
+        ]

helm/benchmark/scenarios/image_generation/cub200_scenario.py

@@ -0,0 +1,95 @@
+import os
+from typing import List
+
+import pandas as pd
+
+from helm.common.media_object import MediaObject, MultimediaObject
+from helm.common.general import ensure_file_downloaded, shell
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, CORRECT_TAG, TEST_SPLIT
+
+
+class CUB200Scenario(Scenario):
+    """
+    Caltech-UCSD Birds-200-2011 (CUB-200-2011) is an extended version of the CUB-200 dataset,
+    a challenging dataset of 200 bird species.
+
+    Number of categories: 200
+    Number of images: 11,788
+    Annotations per image: 15 Part Locations, 312 Binary Attributes, 1 Bounding Box
+
+    Paper: https://authors.library.caltech.edu/27452/1/CUB_200_2011.pdf
+    Website: http://www.vision.caltech.edu/datasets/cub_200_2011
+
+    We use the version from "AttnGAN: Fine-Grained Text to Image Generation with Attentional
+    Generative Adversarial Networks" where 10 captions are included for each image.
+    The sizes of the splits are as follows:
+
+    Train: 8,855 examples
+    Test: 2,933 examples
+
+    Paper: https://arxiv.org/abs/1711.10485
+    Website: https://github.com/taoxugit/AttnGAN
+    """
+
+    IMAGES_DOWNLOAD_URL: str = "https://data.caltech.edu/records/65de6-vp158/files/CUB_200_2011.tgz?download=1"
+    CAPTIONS_DOWNLOAD_URL: str = "https://drive.google.com/uc?export=download&id=1O_LtUP9sch09QH3s_EBAgLEctBQ5JBSJ"
+
+    name = "cub200"
+    description = (
+        "Caltech-UCSD Birds-200-2011 is a challenging dataset of 200 bird species with 10 captions for each bird"
+        "([paper](https://authors.library.caltech.edu/27452/1/CUB_200_2011.pdf), "
+        "[paper](https://arxiv.org/abs/1711.10485))."
+    )
+    tags = ["text-to-image", "image-to-text"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the images
+        images_path: str = os.path.join(output_path, "images")
+        ensure_file_downloaded(
+            source_url=self.IMAGES_DOWNLOAD_URL,
+            target_path=images_path,
+            unpack=True,
+            unpack_type="untar",
+        )
+        images_path = os.path.join(images_path, "CUB_200_2011", "images")
+
+        # Download the captions
+        captions_path: str = os.path.join(output_path, "captions")
+        ensure_file_downloaded(
+            source_url=self.CAPTIONS_DOWNLOAD_URL,
+            target_path=captions_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+        captions_path = os.path.join(captions_path, "birds")
+        text_path: str = os.path.join(captions_path, "text")
+        if not os.path.exists(text_path):
+            shell(["unzip", os.path.join(captions_path, "text.zip"), "-d", captions_path])
+
+        # Get the text examples. Each example has an image file and text file with 10 captions
+        test_filenames_path: str = os.path.join(captions_path, "test", "filenames.pickle")
+        test_filenames: List[str] = pd.read_pickle(test_filenames_path)
+        assert len(test_filenames) == 2_933, "Expected 2,933 examples in the test split."
+
+        instances: List[Instance] = []
+        for file_name in test_filenames:
+            image_path: str = os.path.join(images_path, f"{file_name}.jpg")
+            assert os.path.exists(image_path), f"Expected an image at path: {image_path}"
+
+            caption_path: str = os.path.join(text_path, f"{file_name}.txt")
+            with open(caption_path, "r") as f:
+                captions: List[str] = [caption_line.rstrip() for caption_line in f if caption_line.rstrip()]
+                assert len(captions) == 10, f"Expected 10 captions at path: {caption_path}"
+
+            for caption in captions:
+                content: MultimediaObject = MultimediaObject(
+                    [MediaObject(content_type="image/jpeg", location=image_path)]
+                )
+                instance = Instance(
+                    Input(text=caption),
+                    references=[Reference(Output(multimedia_content=content), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+                instances.append(instance)
+
+        return instances