crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/bird_sql_scenario_helper.py

@@ -0,0 +1,118 @@
+# flake8: noqa
+# type: ignore
+# fmt: off
+
+from typing import Dict
+import os
+import sqlite3
+
+
+# The following code is copied verbatim from:
+# https://github.com/AlibabaResearch/DAMO-ConvAI/blob/90a76ef9ef1e2960c9bdfa38c63cc81b841e813e/bird/llm/src/gpt_request.py
+# under the following license:
+#
+# MIT License
+#
+# Copyright (c) 2022 Alibaba Research
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+
+def get_db_schemas(bench_root: str, db_name: str) -> Dict[str, str]:
+    """
+    Read an sqlite file, and return the CREATE commands for each of the tables in the database.
+    """
+    asdf = 'database' if bench_root == 'spider' else 'databases'
+    with sqlite3.connect(f'file:{bench_root}/{asdf}/{db_name}/{db_name}.sqlite?mode=ro', uri=True) as conn:
+        # conn.text_factory = bytes
+        cursor = conn.cursor()
+        cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
+        tables = cursor.fetchall()
+        schemas = {}
+        for table in tables:
+            cursor.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='{}';".format(table[0]))
+            schemas[table[0]] = cursor.fetchone()[0]
+
+        return schemas
+
+def nice_look_table(column_names: list, values: list):
+    rows = []
+    # Determine the maximum width of each column
+    widths = [max(len(str(value[i])) for value in values + [column_names]) for i in range(len(column_names))]
+
+    # Print the column names
+    header = ''.join(f'{column.rjust(width)} ' for column, width in zip(column_names, widths))
+    # print(header)
+    # Print the values
+    for value in values:
+        row = ''.join(f'{str(v).rjust(width)} ' for v, width in zip(value, widths))
+        rows.append(row)
+    rows = "\n".join(rows)
+    final_output = header + '\n' + rows
+    return final_output
+
+def generate_schema_prompt(db_path, num_rows=None):
+    # extract create ddls
+    '''
+    :param root_place:
+    :param db_name:
+    :return:
+    '''
+    full_schema_prompt_list = []
+    conn = sqlite3.connect(db_path)
+    # Create a cursor object
+    cursor = conn.cursor()
+    cursor.execute("SELECT name FROM sqlite_master WHERE type='table'")
+    tables = cursor.fetchall()
+    schemas = {}
+    for table in tables:
+        if table == 'sqlite_sequence':
+            continue
+        cursor.execute("SELECT sql FROM sqlite_master WHERE type='table' AND name='{}';".format(table[0]))
+        create_prompt = cursor.fetchone()[0]
+        schemas[table[0]] = create_prompt
+        if num_rows:
+            cur_table = table[0]
+            if cur_table in ['order', 'by', 'group']:
+                cur_table = "`{}`".format(cur_table)
+
+            cursor.execute("SELECT * FROM {} LIMIT {}".format(cur_table, num_rows))
+            column_names = [description[0] for description in cursor.description]
+            values = cursor.fetchall()
+            rows_prompt = nice_look_table(column_names=column_names, values=values)
+            verbose_prompt = "/* \n {} example rows: \n SELECT * FROM {} LIMIT {}; \n {} \n */".format(num_rows, cur_table, num_rows, rows_prompt)
+            schemas[table[0]] = "{} \n {}".format(create_prompt, verbose_prompt)
+
+    for k, v in schemas.items():
+        full_schema_prompt_list.append(v)
+
+    schema_prompt = "\n\n".join(full_schema_prompt_list)
+
+    return schema_prompt
+
+def generate_comment_prompt(question, knowledge=None):
+    pattern_prompt_no_kg = "-- Using valid SQLite, answer the following questions for the tables provided above."
+    pattern_prompt_kg = "-- Using valid SQLite and understading External Knowledge, answer the following questions for the tables provided above."
+    # question_prompt = "-- {}".format(question) + '\n SELECT '
+    question_prompt = "-- {}".format(question)
+    knowledge_prompt = "-- External Knowledge: {}".format(knowledge)
+
+    if not knowledge_prompt:
+        result_prompt = pattern_prompt_no_kg + '\n' + question_prompt
+    else:
+        result_prompt = knowledge_prompt + '\n' + pattern_prompt_kg + '\n' + question_prompt
+
+    return result_prompt
+
+def cot_wizard():
+    cot = "\nGenerate the SQL after thinking step by step: "
+    
+    return cot

helm/benchmark/scenarios/blimp_scenario.py

@@ -4,7 +4,7 @@ from typing import List
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .scenario import CORRECT_TAG, Reference, Scenario, Instance, Input, TEST_SPLIT, Output
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, Reference, Scenario, Instance, Input, TEST_SPLIT, Output
 
 
 class BLiMPScenario(Scenario):

helm/benchmark/scenarios/bluex_scenario.py

@@ -0,0 +1,70 @@
+from typing import Any, List
+from pathlib import Path
+from datasets import load_dataset
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class BLUEXScenario(Scenario):
+    """
+    The BLUEX dataset is a benchmark used for evaluating natural language processing models in Brazilian Portuguese.
+    It consists of multiple-choice questions taken from official entrance exams of Unicamp (Convest) and USP (Fuvest),
+    covering various high school subjects. The questions include both textual prompts and visual elements. This dataset
+    was developed to assess the performance of models on tasks involving comprehension and reasoning, with a specific
+    focus on texts and exams originally written in Portuguese.
+    """
+
+    name = "bluex"
+    description = "MQA benchmark with questions from Brazilian entrance exams"
+    tags = ["knowledge", "multiple_choice", "pt-br"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data and read all the dialogues
+        dataset: Any
+        # Read all the instances
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+
+        dataset = load_dataset(
+            "portuguese-benchmark-datasets/BLUEX",
+            revision="d99cf6d05b50db7c42a605e5e2924cbd46f076c7",
+            cache_dir=cache_dir,
+        )
+        for example in dataset["questions"]:
+            # This scenario disregards issues with images
+            if example["has_associated_images"]:
+                continue
+            question = example["question"]
+            choices = example["alternatives"]
+            answer = example["answer"]
+
+            answers_dict = {}
+            for alt in choices:
+                if ")" in alt:
+                    label, text = alt.split(")", 1)
+                    label = label.strip().upper()
+                    text = text.strip()
+                    answers_dict[label] = text
+
+            if answer not in answers_dict:
+                continue
+
+            correct_answer = answers_dict[answer]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+            instance = Instance(
+                input=Input(text=question),
+                split=TEST_SPLIT,
+                references=[answer_to_reference(text) for text in answers_dict.values()],
+            )
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/bold_scenario.py

@@ -3,14 +3,16 @@ import os
 import random
 from typing import List, Dict, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     TRAIN_SPLIT,
     TEST_SPLIT,
     DEFAULT_TEST_SIZE,
     Input,
+    ScenarioMetadata,
 )
 
 TOXIC_TAG = "toxic"
@@ -98,7 +100,7 @@ class BOLDScenario(Scenario):
                     # with https://raw.githubusercontent.com/amazon-
                     # research/bold/main/prompts/political_ideology_prompt.json as an example:
                     # {"left-wing": {"left-wing_politics": }}
-                    for (prompt_category, prompt_topic) in prompt_topics_dict.items():
+                    for prompt_category, prompt_topic in prompt_topics_dict.items():
                         for prompt_text in prompt_topic.values():
                             prompt_text = prompt_text[0]  # stored as a list containing a single string
                             prompt_tuples.append((prompt_category, prompt_text))
@@ -108,7 +110,7 @@ class BOLDScenario(Scenario):
 
         split_sizes = {"train": len(instances) - DEFAULT_TEST_SIZE, "test": DEFAULT_TEST_SIZE}
 
-        for (idx, prompt_tuple) in enumerate(prompt_tuples):
+        for idx, prompt_tuple in enumerate(prompt_tuples):
             prompt_category, prompt_text = prompt_tuple
             curr_split = TRAIN_SPLIT
 
@@ -118,3 +120,16 @@ class BOLDScenario(Scenario):
             instances.append(Instance(Input(text=f"{prompt_text} "), split=curr_split, references=[]))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="bold",
+            display_name="BOLD (Bias in Open-Ended Language Generation Dataset)",
+            short_display_name="BOLD",
+            description="The Bias in Open-Ended Language Generation Dataset (BOLD) for measuring biases "
+            "and toxicity in open-ended language generation [(Dhamala et al., "
+            "2021)](https://dl.acm.org/doi/10.1145/3442188.3445924).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="unknown",
+            main_split="test",
+        )

helm/benchmark/scenarios/boolq_scenario.py

@@ -2,8 +2,9 @@ import json
 import os
 from typing import List, Dict, Optional
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,
@@ -13,6 +14,7 @@ from .scenario import (
     PassageQuestionInput,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -188,3 +190,21 @@ class BoolQScenario(Scenario):
             split_path: str = os.path.join(data_path, filename)
             instances.extend(self.get_split_instances(split, split_path, contrast_map if split == VALID_SPLIT else {}))
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="boolq",
+            display_name="BoolQ",
+            short_display_name=None,
+            description="The BoolQ benchmark for binary (yes/no) question answering [(Clark et al., "
+            "2019)](https://aclanthology.org/N19-1300/).",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="passages from Wikipedia, questions from search queries",
+                when="2010s",
+                who="web users",
+                language="English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="valid",
+        )

helm/benchmark/scenarios/call_center_scenario.py

@@ -0,0 +1,84 @@
+import datasets
+import os
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Output,
+    Reference,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    Input,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class CallCenterSummarizationScenario(Scenario):
+    """Call center summarization."""
+
+    name = "call_center_summarization"
+    description = "Call center summarization."
+    tags = ["call_center"]
+
+    def __init__(self, subset: str):
+        super().__init__()
+        self.subset = subset
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset("yifanmai/call-center", self.subset, split="test", cache_dir=cache_dir)
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=row["transcript"])
+            instance = Instance(input=input, references=[], split=TEST_SPLIT)
+            instances.append(instance)
+        return instances
+
+
+class CallCenterSummarizationPairwiseComparisonScenario(Scenario):
+    """Call center summarization."""
+
+    name = "call_center_summarization_pairwise_comparison"
+    description = "Call center summarization."
+    tags = ["call_center"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "yifanmai/call-center", "summarization_with_annotations", split="test", cache_dir=cache_dir
+        )
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=row["transcript"])
+            reference = Reference(output=Output(text=row["gpt-4o-mini-2024-07-18_summary"]), tags=[CORRECT_TAG])
+            instance = Instance(input=input, references=[reference], split=TEST_SPLIT)
+            instances.append(instance)
+        return instances
+
+
+class CallCenterSummarizationKeyPointsRecallScenario(Scenario):
+    """Call center summarization."""
+
+    name = "call_center_summarization_key_points_recall"
+    description = "Call center summarization."
+    tags = ["call_center"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "yifanmai/call-center", "summarization_with_annotations", split="test", cache_dir=cache_dir
+        )
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=row["transcript"])
+            references = [
+                Reference(output=Output(text=key_point), tags=[CORRECT_TAG])
+                for key_point in row["gpt-4o-mini-2024-07-18_key_points"]
+            ]
+            instance = Instance(input=input, references=references, split=TEST_SPLIT)
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/casehold_scenario.py

@@ -0,0 +1,79 @@
+from typing import List
+import os
+import os.path
+
+from datasets import load_dataset, DatasetDict
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Output,
+)
+
+
+class CaseHOLDScenario(Scenario):
+    """
+     CaseHOLD QA
+       CaseHOLD is a multiple choice question answering task derived from legal citations in judicial rulings.
+       CaseHOLD consists of ~53,000 questions, mined from the Harvard Law Library case law corpus.
+
+     Dataset repository
+       https://huggingface.co/datasets/casehold/casehold
+     Publication
+       "When Does Pretraining Help? Assessing Self-Supervised Learning for Law and the CaseHOLD Dataset"
+       ICAIL, 2021
+       https://reglab.stanford.edu/data/casehold-benchmark/
+       https://arxiv.org/abs/2104.08671
+
+    Data content
+      The citing context from the judicial decision serves as the prompt for the question.
+      The answer choices are holding statements derived from citations following text in a legal decision.
+      There are five answer choices for each citing text.
+      The correct answer is the holding statement that corresponds to the citing text.
+      The four incorrect answers are other holding statements.
+
+    """  # noqa: E501
+
+    name = "casehold"
+    description = "CaseHOLD (Case Holdings On Legal Decisions) is a multiple choice question answering scenario where the task is to identify the relevant holding of a cited case [(Zheng et al, 2021)](https://arxiv.org/pdf/2104.08671.pdf)."  # noqa: E501
+    tags = ["question_answering", "legal"]
+
+    # Note: Skip the validation split since we don't need it
+    HELM_SPLIT_NAME_TO_DATASETS_SPLIT_NAME = {TRAIN_SPLIT: "train", TEST_SPLIT: "test"}
+    NUM_REFERENCES = 5
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path: str = os.path.join(output_path, "data")
+        ensure_directory_exists(data_path)
+        dataset: DatasetDict = load_dataset(
+            "casehold/casehold",
+            "all",
+            cache_dir=data_path,
+        )
+
+        instances: List[Instance] = []
+        for helm_split_name, datasets_split_name in self.HELM_SPLIT_NAME_TO_DATASETS_SPLIT_NAME.items():
+            split_data = dataset[datasets_split_name]
+            for example in split_data:
+                example_id = example["example_id"]
+                citing_prompt = example["citing_prompt"]
+                holdings = [example[f"holding_{i}"] for i in range(self.NUM_REFERENCES)]
+                correct_label: str = example["label"]
+                references = [
+                    Reference(Output(text=holdings[i]), tags=([CORRECT_TAG] if correct_label == str(i) else []))
+                    for i in range(self.NUM_REFERENCES)
+                ]
+                instance = Instance(
+                    input=Input(text=citing_prompt),
+                    references=references,
+                    split=helm_split_name,
+                    id=f"id{example_id}",
+                )
+                instances.append(instance)
+        return instances

helm/benchmark/scenarios/chw_care_plan_scenario.py

@@ -0,0 +1,129 @@
+import pandas as pd
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import check_file_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+
+
+def create_prompt_text(clinical_note):
+    # Create a prompt for the model to generate a care plan based on the clinical note
+    prompt = f"""
+You are provided with a clinical note regarding a physician-patient interaction. Your task is to \
+extract specific information based solely on the content provided. Do not hallucinate or infer details \
+that are not explicitly stated in the text. Any information you include must be directly entailed by the text.
+
+Instructions:
+
+Extract the required information precisely as presented in the source text.
+
+If the text does not contain specific information, clearly state "Not mentioned".
+
+Maintain the patient's original wording whenever possible.
+
+Response Format:
+
+Chief Complaint
+[ENTER CHIEF COMPLAINT]
+
+History of Present Illness
+
+Onset: When did it start? Did it begin suddenly or gradually? [ENTER ONSET INFORMATION]
+
+Provoking/Palliating Factors: What makes the symptoms better or worse? [ENTER Provoking/Palliating Factors INFORMATION]
+
+Quality: Describe the symptoms, e.g., sharp pain, dull pain, stabbing pain. [ENTER QUALITY INFORMATION]
+
+Region/Radiation: Where are your symptoms located? Do they move? [ENTER REGION/RADIATION INFORMATION]
+
+Severity: On a scale of 1 to 10, how severe are your symptoms? [ENTER SEVERITY INFORMATION]
+
+Timing: When do you experience the symptoms? What times of day? [ENTER TIMING INFORMATION]
+
+Related Symptoms: Are there any other symptoms related to the main complaint? [ENTER RELATED SYMPTOMS INFORMATION]
+
+Ensure your responses are concise, accurate, and entirely supported by the provided text. \
+Do not introduce external knowledge or assumptions.
+
+Clinical Note:
+{clinical_note}
+"""
+    return prompt
+
+
+class CHWCarePlanScenario(Scenario):
+    """
+    A scenario for a dataset containing free form text of a clinical health worker care plan, with the
+    associated goal being to restructure that text into a given format.
+
+    - Input:  The clinical note (column "MO Note").
+    - Output:  The clinical note (column "MO Note"). We will use this note as the reference for entailment.
+    """
+
+    name = "chw_care_plan"
+    description = (
+        "NoteExtract is a benchmark that focuses on the structured extraction of information"
+        "from free-form clinical text. It provides care plan notes authored by health workers"
+        "and evaluates a model's ability to convert them into a predefined structured format,"
+        "such as fields for Chief Complaint and History of Present Illness. The benchmark"
+        "emphasizes faithful extraction without hallucination or inference."
+    )
+    tags = ["question_answering", "biomedical"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[CHWCarePlanScenario] Required data file not found: '{self.data_path}'")
+        df = pd.read_csv(self.data_path)  # columns: ["text", "target", ...]
+
+        instances: List[Instance] = []
+
+        # Use the entire dataset as one split (TEST_SPLIT)
+        for idx, row in df.iterrows():
+            note_text: str = row["MO Note"]
+            prompt_text = create_prompt_text(note_text)
+            if pd.isna(note_text):
+                print(f"Skipping row {idx} due to NaN value in 'MO Note'")
+                continue
+            # print(f"Prompt text: {prompt_text}")
+
+            # Create one Instance per patient
+            instance = Instance(
+                input=Input(text=prompt_text),
+                references=[Reference(Output(text=note_text), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="chw_care_plan",
+            display_name="NoteExtract",
+            description="NoteExtract is a benchmark that focuses on the structured extraction of "
+            "information from free-form clinical text. It provides care plan notes authored "
+            "by health workers and evaluates a model's ability to convert them into a "
+            "predefined structured format, such as fields for Chief Complaint and History "
+            "of Present Illness. The benchmark emphasizes faithful extraction without "
+            "hallucination or inference.",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Convert general text care plans into structured formats",
+                when="Any",
+                who="Clinician, Researcher",
+                language="English",
+            ),
+            main_metric="chw_care_plan_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/ci_mcqa_scenario.py

@@ -0,0 +1,80 @@
+import json
+import os
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    Input,
+    Output,
+)
+
+
+class CIMCQAScenario(Scenario):
+    """CIMCQA is a multiple-choice question answering (MCQA) dataset designed to
+    study concept inventories in CS Education.
+
+    This is used by a pre-publication paper.
+
+    NOTE: This code is for archival purposes only. The scenario cannot be run because it requires
+    private data. Please contact the paper authors for more information."""
+
+    DATASET_DOWNLOAD_URL: str = "https://drive.google.com/uc?export=download&id=1siYjhDiasI5FIiS0ckLbo40UnOj8EU2h"
+
+    name = "ci_mcqa"
+    description = (
+        "CIMCQA is a multiple-choice question answering (MCQA) dataset designed to"
+        "study concept inventories in CS Education."
+    )
+    tags = ["question_answering"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path: str = os.path.join("restricted", "bdsi_multiple_answers_removed.json")
+        assert os.path.exists(data_path)
+
+        with open(data_path, "r", encoding="utf8") as f:
+            data = json.load(f)
+
+        # Data is a list of dictionaries now, each one a question and its associated answers and metadata.
+        instances: List[Instance] = list()
+
+        # UNCOMMENT BELOW FOR FEW-SHOT RUN
+        training_data_path: str = os.path.join("restricted", "mock_bdsi_multiple_answers_removed.json")
+        assert os.path.exists(training_data_path)
+
+        with open(training_data_path, "r", encoding="utf8") as f:
+            training_data = json.load(f)
+        for question in training_data:
+            question_text = question["question"]
+            references = list()
+            for index, answer in enumerate(question["options"]):
+                reference_answer = Output(text=answer)
+                # Correct option offset by 1 due to zero-indexing
+                tag = [CORRECT_TAG] if index == question["correct_option"] - 1 else []
+                references.append(Reference(reference_answer, tags=tag))
+            instance = Instance(
+                input=Input(text=question_text),
+                references=references,
+                split=TRAIN_SPLIT,
+            )
+            instances.append(instance)
+
+        for question in data:
+            question_text = question["question"]
+            references = list()
+            for index, answer in enumerate(question["options"]):
+                reference_answer = Output(text=answer)
+                # Correct option offset by 1 due to zero-indexing
+                tag = [CORRECT_TAG] if index == question["correct_option"] - 1 else []
+                references.append(Reference(reference_answer, tags=tag))
+            instance = Instance(
+                input=Input(text=question_text),
+                references=references,
+                split=TEST_SPLIT,  # Just doing zero shot to start
+            )
+            instances.append(instance)
+        return instances