crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/scenarios/shc_bmt_scenario.py

@@ -0,0 +1,69 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCBMTMedScenario(Scenario):
+    """
+    This benchmark dataset was built from a patient status gold-standard
+    for specific questions asked after a bone marrow transplant has taken place.
+    """
+
+    name = "shc_bmt_med"
+    description = (
+        "A dataset containing patient notes with associated "
+        "questions and answers related to bone marrow transplantation."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f"Provide an answer to the following {question} with the following context: {context} "
+                    ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
+                    "details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-BMT-dataset_filtered.csv"
+
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCBMTMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCBMTMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_cdi_scenario.py

@@ -0,0 +1,70 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCCDIMedScenario(Scenario):
+    """
+    This benchmark dataset was built from Clinical Document Integrity (CDI)
+    notes were there are verifications of clinical activities. The idea behind
+    it was to assess an LLM capability to answer these questions from previous notes.
+    """
+
+    name = "shc_cdi_med"
+    description = (
+        "A dataset built from Clinical Document Integrity (CDI) notes, to assess "
+        "the ability to answer verification questions from previous notes."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f"Provide an answer to the following {question} with the following context: {context} , "
+                    "Answer the question with either 'A' for yes or 'B' for no. Do not provide any "
+                    "additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-CDI-dataset_filtered.csv"
+
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCCDIMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCCDIMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_conf_scenario.py

@@ -0,0 +1,70 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCCONFMedScenario(Scenario):
+    """
+    Benchmark derived from extracting confidential information from clinical notes.
+    From Evaluation of a Large Language Model to Identify Confidential Content in
+    Adolescent Encounter Notes published at https://jamanetwork.com/journals/jamapediatrics/fullarticle/2814109
+    """
+
+    name = "shc_conf_med"
+    description = (
+        "A dataset of clinical notes from adolescent patients used to identify sensitive "
+        "protected health information that should be restricted from parental access."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f"Provide an answer to the following {question} with the following context: {context} "
+                    ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
+                    "details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-CONF-dataset_filtered.csv"
+
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCCONFMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCCONFMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_ent_scenario.py

@@ -0,0 +1,72 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCENTMedScenario(Scenario):
+    """
+    This benchmark dataset was built to assess the capabilities "
+    "of an LLM for referral to the Ear, Nose and Throat department.
+    """
+
+    name = "shc_ent_med"
+    description = (
+        "A dataset designed to evaluate performance in "
+        "identifying appropriate patient referrals to Ear, Nose, and Throat specialists."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B", "C"]
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        counter = 1
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                if row["label"] != "":  # skip rows with character/encoding issues - 79
+                    question = row["prompt"]
+                    context = row["context"]
+                    answer = row["label"]
+                    prompt = (
+                        f"{counter} Provide an answer to the following {question} with the following context:"
+                        f" {context} , Answer the question with either 'A' for yes, 'B' for no, or 'C' for no mention."
+                        " Do not provide any additional details or response, just a simple A, B, or C response."
+                    )
+                    data[prompt] = answer
+                    counter = counter + 1
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-ENT-dataset_filtered.csv"
+
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCENTMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCENTMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_gip_scenario.py

@@ -0,0 +1,66 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCGIPMedScenario(Scenario):
+    """
+    This benchmark dataset was built from a patient referral gold-standard set
+    to a specialty clinic to verify the ability of LLMs for patient hospice referral purposes.
+    """
+
+    name = "shc_gip_med"
+    description = "A dataset evaluating performance in identifying appropriate patient referrals to hospice care."
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f"Provide an answer to the following {question} with the following context: {context} "
+                    ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
+                    "details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-GIP-dataset_filtered.csv"
+
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCGIPMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCGIPMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_ptbm_scenario.py

@@ -0,0 +1,76 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCPTBMMedScenario(Scenario):
+    """
+    This dataset contains clinical notes from primary care visit encounters of
+    children ages 4-6 years old with ADHD seen at Stanford's community-based primary
+    care network, Packard Children's Health Alliance, between 2015-2019. In this classification
+    task, the LLM is tasked with classifying whether the note contains clinician recommendation
+    for parent training in behavior management, which is the first-line evidence-based treatment
+    for young children with ADHD. From publication: https://doi.org/10.1093/jamia/ocae001
+    """
+
+    name = "shc_ptbm_med"
+    description = (
+        "A dataset that classifies whether a clinical note contains a clinician "
+        "recommendation for parent training in behavior management, which is the first-line "
+        "evidence-based treatment for young children with ADHD."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    "You are reviewing a clinical note from health records of children with "
+                    "attention deficit hyperactivity disorder (ADHD) and classifying mentions of "
+                    f"behavioral therapy. Provide an answer to the following {question} with the "
+                    f"following context: {context} , Answer the question with a 'A' for yes or 'B' "
+                    "for no. Do not provide any additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-PTBM-dataset_filtered.csv"
+
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCPTBMMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCPTBMMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_sei_scenario.py

@@ -0,0 +1,89 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCSEIMedScenario(Scenario):
+    """
+    This dataset contains clinical notes from primary care visit encounters
+    (in-person/telehealth and telephone) of children ages 6-11 years old with ADHD
+    seen at Stanford's community-based primary care network, Packard Children's Health Alliance,
+    between 2015-2022. All children in this dataset were prescribed at least once an ADHD
+    medication (stimulants or non-stimulants) by a primary care clinician. In this
+    classification task, the LLM is tasked with classifying whether the note contains
+    documentation of side effect monitoring (recording of absence or presence of
+    medication side effects), as recommended in clinical practice guidelines.
+    From publication: https://doi.org/10.1542/peds.2024-067223
+    """
+
+    name = "shc_sei_med"
+    description = (
+        "A dataset that classifies whether a clinical note contains documentation "
+        "of side effect monitoring (recording of absence or presence of medication "
+        "side effects), as recommended in clinical practice guidelines."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    "You are reviewing a clinical note from health records of children "
+                    "with attention deficit hyperactivity disorder (ADHD). Given the following "
+                    "definitions: side Effects Inquiry (SEI): Explicit documentation by the clinician "
+                    "asking about current side effects related to ADHD medications that the child is "
+                    "taking or documentation of specific ADHD medication side effects experienced "
+                    "by the patient. SEI does *not* include future side effects monitoring, "
+                    "such as documentation of potential ADHD medication side effects, including "
+                    "planning to follow patients to monitor side effects, explaining about "
+                    "potential side effects of an ADHD medication. These documentations are not "
+                    "categorized as SEI because they consist of a plan or an explanation about "
+                    "side effects without actual side effect monitoring taking place, and "
+                    "No Side Effects Inquiry (NSEI): No evidence of side effects monitoring. "
+                    f"Provide an answer to the following {question} with the following context: {context} "
+                    ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
+                    "details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-SEI-dataset_filtered.csv"
+
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCSEIMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCSEIMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_sequoia_scenario.py

@@ -0,0 +1,69 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCSequoiaMedScenario(Scenario):
+    """
+    Benchmark derived from manually curated answers to several questions for Sequoia clinic referrals
+    """
+
+    name = "shc_sequoia_med"
+    description = (
+        "A dataset containing manually curated answers to questions regarding patient referrals to the Sequoia clinic."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        counter = 1
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)  # , quoting=csv.QUOTE_MINIMAL
+            for row in reader:
+                question = row["question"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f" {counter} Provide an answer to the following {question} with the following context:"
+                    f" {context} , Answer the question with a 'A' for yes or 'B' for no. Do not provide any "
+                    "additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+                counter += 1
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-sequoia-dataset_filtered.csv"
+
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCSequoiaMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCSequoiaMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/simple_safety_tests_scenario.py

@@ -1,7 +1,7 @@
 from typing import List
 from datasets import load_dataset
 
-from .scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output
 
 
 class SimpleSafetyTestsScenario(Scenario):

helm/benchmark/scenarios/spider_scenario.py

@@ -0,0 +1,91 @@
+import json
+import os
+from typing import Dict, List
+
+from filelock import FileLock
+
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded, shell
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.bird_sql_scenario_helper import (  # type: ignore
+    generate_schema_prompt,
+)
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Scenario,
+    Instance,
+    Reference,
+    VALID_SPLIT,
+    Input,
+    Output,
+)
+
+
+def _ensure_file_unzipped(source_path: str, target_path: str):
+    with FileLock(f"{target_path}.lock"):
+        if os.path.exists(target_path):
+            hlog(f"Not decompressing {source_path} because {target_path} already exists")
+            return
+        tmp_path = target_path + ".tmp"
+        ensure_directory_exists(tmp_path)
+        shell(["unzip", source_path, "-d", tmp_path])
+        shell(["mv", tmp_path, target_path])
+
+
+class SpiderScenario(Scenario):
+    """Spider 1.0"""
+
+    name = "spider"
+    description = "spider"
+    tags = ["sql"]
+
+    INSTRUCTIONS_PROMPT = """-- Using valid SQLite, answer the following questions for the tables provided above.
+"""
+    COT_PROMPT = """
+Think step by step, then generate a single SQL query in valid SQLite syntax. Respond with only your reasoning and SQL query in the following tag-delimited format:
+
+<reasoning>
+INSERT_YOUR_REASONING_HERE
+</reasoning>
+<sql>
+INSERT_YOUR_SQL_QUERY_HERE
+</sql>"""  # noqa: E501
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_parent_path = os.path.join(output_path, "data")
+        ensure_file_downloaded(
+            "https://drive.google.com/uc?id=1403EGqzIDoHMdQF4c9Bkyl7dZLZ5Wt6J&export=download&confirm=t",
+            data_parent_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+        data_root_path = os.path.join(data_parent_path, "spider_data")
+        databases_root_path = os.path.join(data_root_path, "test_database")
+
+        database_schema_prompts: Dict[str, str] = {}
+        for database_name in os.listdir(databases_root_path):
+            database_path = os.path.join(databases_root_path, database_name, f"{database_name}.sqlite")
+            if not os.path.exists(database_path):
+                # Ignore stray ".DS_Store" directory
+                continue
+
+            database_schema_prompt = generate_schema_prompt(database_path, num_rows=None)
+            database_schema_prompts[database_name] = database_schema_prompt
+
+        instances: List[Instance] = []
+        dataset_path = os.path.join(data_root_path, "test.json")
+        dataset = json.load(open(dataset_path, "r"))
+        for row in dataset:
+            database_id: str = row["db_id"]
+            question: str = row["question"]
+            gold_sql: str = row["query"]
+
+            schema_prompt = database_schema_prompts[database_id]
+            combined_prompt = schema_prompt + "\n\n" + self.INSTRUCTIONS_PROMPT + question + self.COT_PROMPT
+            instance = Instance(
+                input=Input(text=combined_prompt),
+                references=[Reference(output=Output(text=gold_sql), tags=[CORRECT_TAG])],
+                extra_data={"db_id": row["db_id"]},
+                split=VALID_SPLIT,
+            )
+            instances.append(instance)
+        return instances