crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/decodingtrust_toxicity_prompts_scenario.py

@@ -0,0 +1,90 @@
+import json
+import os
+import random
+from typing import List, Dict
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import ensure_file_downloaded
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
+
+
+DATA_REPO_HASH = "38972f6ccbf376a8d0660babafb4d2b3b9cca3f4"
+
+
+class DecodingTrustToxicityPromptsScenario(Scenario):
+    """
+    The DecodingTrustToxicityPrompts dataset is from the paper:
+    https://arxiv.org/abs//2306.11698
+
+    The dataset contains 99,016 naturally occurring prompts (21,744 toxic (22%) and 77,272 non-toxic prompts (78%)).
+    The authors sampled ~25,000 sentences from four equal width toxicity ranges: [[0, 0.25), ..., [0.75, 1]).
+    Sentences are split in half, producing a prompt and a continuation.
+    """
+
+    name = "decodingtrust_toxicity_prompts"
+    description = "Toxicity analysis of LM generations when fed naturally occurring, sentence-level prompts."
+    tags = ["harms", "toxicity"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        self.subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        source_url = (
+            f"https://raw.githubusercontent.com/AI-secure/DecodingTrust-Data-Legacy/{DATA_REPO_HASH}/data/toxicity/"
+            f"user_prompts/{{}}.jsonl"
+        )
+
+        TOXIC_SUB_SPLIT: str = "toxic"
+        NONTOXIC_SUB_SPLIT: str = "non-toxic"
+        GPT3_SUB_SPLIT: str = "gpt-3.5"
+        GPT4_SUB_SPLIT: str = "gpt-4"
+        FILENAMES = {
+            TOXIC_SUB_SPLIT: "toxic",
+            NONTOXIC_SUB_SPLIT: "nontoxic",
+            GPT3_SUB_SPLIT: "toxic.jsonl.gpt3.n=25.out1.split.annotated.jsonl.filtered",
+            GPT4_SUB_SPLIT: "toxic.jsonl.gpt4.n=25.out1.split.annotated.jsonl.filtered",
+        }
+
+        source_url = source_url.format(FILENAMES[self.subject])
+        data_path = os.path.join(output_path, FILENAMES[self.subject])
+
+        ensure_file_downloaded(
+            source_url=source_url,  # to be filled
+            target_path=data_path,
+        )
+
+        instances: List[Instance] = []
+
+        prompts_path: str = data_path
+        with open(prompts_path, "r") as f:
+            for line in f:
+                prompt: Dict = json.loads(line)["prompt"]
+                prompt_text: str = prompt["text"]
+
+                # From the paper, "we label a prompt as toxic if it has TOXICITY ≥ 0.5, and non-toxic otherwise."
+                instance: Instance = Instance(
+                    input=Input(text=prompt_text),
+                    references=[],
+                    split=TEST_SPLIT,
+                )
+                instances.append(instance)
+
+        # The dataset has a high proportion of non-toxic prompts at the beginning
+        # (only one toxic prompt in the first couple hundred).
+        # Shuffle, so we get a mix of toxic and non-toxic prompts when we're not evaluating all the prompts.
+        random.seed(0)
+        random.shuffle(instances)
+
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="decodingtrust_toxicity_prompts",
+            display_name="DecodingTrust - Toxicity",
+            short_display_name="Toxicity",
+            description="Evaluation of the privacy understanding and privacy preserving properties of " "LLMs",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="unknown",
+            main_split="test",
+        )

helm/benchmark/scenarios/dialogue_scenarios.py

@@ -6,7 +6,18 @@ from typing import Dict, List, Optional
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.runner import get_benchmark_output_path
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 @dataclass(frozen=True)
@@ -79,7 +90,6 @@ class EmpatheticDialoguesScenario(Scenario):
             # Group rows by prompts, each group corresponds to an instance
             grouped_data_df = data_df.groupby(by=["prompt", "context"])
             for prompt_cols, prompt_df in grouped_data_df:
-
                 # Group rows by conversations, each group corresponds to a reference
                 grouped_prompt_df = prompt_df.groupby(["conv_id", "selfeval"])
                 references = []
@@ -139,5 +149,5 @@ class EmpatheticDialoguesScenario(Scenario):
 
 if __name__ == "__main__":
     scenario = EmpatheticDialoguesScenario()
-    instances = scenario.get_instances("./benchmark_output/scenarios/empatheticdialogues")
+    instances = scenario.get_instances(os.path.join(get_benchmark_output_path(), "scenarios/empatheticdialogues"))
     print(instances[100])

helm/benchmark/scenarios/dischargeme_scenario.py

@@ -0,0 +1,196 @@
+from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import check_file_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+import pandas as pd
+
+
+def file_preprocessing(data_path: str, task_objective: str) -> pd.DataFrame:
+    """
+    Preprocess the data files to create a DataFrame with the necessary columns.
+    task_objective: 'brief_hospital_course' or 'discharge_instructions'
+    Use command to download: wget -r -N -c -np --user {PHYSIONET_USERNAME} \
+    --ask-password https://physionet.org/files/discharge-me/1.3/
+    data_path is directory that contains the downloaded files: '{base_dir}/physionet.org/'
+    """
+    # Load the first CSV file
+    diagnosis_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/diagnosis.csv.gz"
+    check_file_exists(
+        diagnosis_path, msg=f"[DischargeMeScenario] Required diagnosis file not found: '{diagnosis_path}'"
+    )
+    discharge_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/discharge.csv.gz"
+    check_file_exists(
+        discharge_path, msg=f"[DischargeMeScenario] Required discharge file not found: '{discharge_path}'"
+    )
+    target_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/discharge_target.csv.gz"
+    check_file_exists(target_path, msg=f"[DischargeMeScenario] Required target file not found: '{target_path}'")
+    radiology_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/radiology.csv.gz"
+    check_file_exists(
+        radiology_path, msg=f"[DischargeMeScenario] Required radiology file not found: '{radiology_path}'"
+    )
+    ed_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/edstays.csv.gz"
+    check_file_exists(ed_path, msg=f"[DischargeMeScenario] Required ed file not found: '{ed_path}'")
+    triage_path = f"{data_path}/files/discharge-me/1.3/test_phase_1/triage.csv.gz"
+    check_file_exists(triage_path, msg=f"[DischargeMeScenario] Required triage file not found: '{triage_path}'")
+    df_diagnosis = pd.read_csv(diagnosis_path, compression="gzip", keep_default_na=False)
+    df_discharge = pd.read_csv(discharge_path, compression="gzip", keep_default_na=False)
+    df_target = pd.read_csv(
+        target_path,
+        compression="gzip",
+        keep_default_na=False,
+    )
+    df_radiology = pd.read_csv(radiology_path, compression="gzip", keep_default_na=False)
+    df_ed = pd.read_csv(ed_path, compression="gzip", keep_default_na=False)
+    df_triage = pd.read_csv(triage_path, compression="gzip", keep_default_na=False)
+    df_diagnosis_triage = pd.merge(
+        df_diagnosis, df_triage, on="subject_id", how="inner", suffixes=("_df_diagnosis", "_df_triage")
+    )
+    df_diagnosis_triage_discharge = pd.merge(
+        df_diagnosis_triage, df_discharge, on="subject_id", how="inner", suffixes=("", "_df_discharge")
+    )
+    df_diagnosis_triage_discharge_radiology = pd.merge(
+        df_diagnosis_triage_discharge, df_radiology, on="hadm_id", how="inner", suffixes=("", "_df_radiology")
+    )
+
+    df_features = pd.merge(
+        df_diagnosis_triage_discharge_radiology, df_ed, on="hadm_id", how="inner", suffixes=("", "_df_ed")
+    )
+
+    # Reduce the DataFrame to remove duplicate hadm_id
+    df_features_reduced = df_features.drop_duplicates(subset="hadm_id")
+    columns_to_keep = ["text", "text_df_radiology", "hadm_id"]
+    df_input = df_features_reduced[columns_to_keep]
+    final_df = pd.merge(df_input, df_target, on="hadm_id", how="inner")
+
+    def remove_substring(string, substring):
+        return string.replace(substring, "")
+
+    final_df["text"] = final_df.apply(lambda row: remove_substring(row["text"], row[task_objective]), axis=1)
+    return final_df
+
+
+def create_prompt(text: str, text_df_radiology: str, task_objective: str) -> str:
+    """
+    Create the prompt for the instance.
+    """
+    prompt = f"Generate the {task_objective} from the following patient discharge text and radiology report text.\
+    \n\nDischarge Text:\n{text}\n\nRadiology Report:\n{text_df_radiology}\n\n{task_objective}:\n"
+    return prompt
+
+
+class DischargeMeScenario(Scenario):
+    """
+    DischargeMe is a discharge instruction generation dataset and brief hospital course generation \
+    dataset collected from MIMIC-IV data.
+    In this scenario, we only consider the discharge text as well as the radiology report text.
+    We are using the phase I test set which is composed of 14,702 hospital admission instances.
+
+    The splits are provided by the dataset itself.
+
+    TASKS = {discharge instruction, brief hospital course}
+    Sample Synthetic Prompt:
+        Generate the {TASK} from the following patient discharge text and radiology report text.
+
+        Discharge Text:
+        Name: {Patient Name} Unit No: {Unit Number} Date of Birth: {DOB} Date of Admission:
+        {DOA} Date of Discharge: {DOD}
+        Chief Complaint: {Chief Complaint} History of Present Illness: {HPI} Past Medical History: {PMH}
+        Medications on Admission: {Medications} Allergies: {Allergies} Physical Exam: {Physical Exam}
+        Discharge Diagnosis: {Discharge Diagnosis}
+
+        Radiology Report:
+        {Radiology Report}
+
+        {TASK}:
+    @inproceedings{Xu_2024,
+        title={ Discharge me: Bionlp acl’24 shared task on streamlining discharge documentation.},
+        url={https://doi.org/10.13026/4a0k-4360},
+        DOI={10.13026/27pt-1259},
+        booktitle={ Proceedings of the 23rd Workshop on Biomedical Natural Language Processing (BioNLP) at ACL 2024},
+        publisher={Association for Computational Linguistics},
+        author={Xu, Justin and Delbrouck, Jean-Benoit and Johnston, Andrew and Blankemeier, Louis and Langlotz, Curtis},
+        year={2024}
+    }
+    """
+
+    name = "dischargeme"
+    description = (
+        "DischargeMe is a benchmark designed to evaluate clinical text generation. It pairs"
+        "discharge summaries and radiology reports from MIMIC-IV with generation tasks"
+        "such as writing discharge instructions or summarizing the brief hospital course. The"
+        "benchmark assesses a model's ability to generate patient-facing documentation that is"
+        "complete, empathetic, and clinically accurate."
+    )
+    tags = ["biomedical"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        df_bhc = file_preprocessing(self.data_path, "brief_hospital_course")
+        df_di = file_preprocessing(self.data_path, "discharge_instructions")
+
+        for i in range(df_bhc.shape[0]):
+            prompt_bhc = create_prompt(
+                df_bhc.iloc[i]["text"], df_bhc.iloc[i]["text_df_radiology"], "Brief Hospital Course"
+            )
+            prompt_di = create_prompt(
+                df_di.iloc[i]["text"], df_di.iloc[i]["text_df_radiology"], "Discharge Instructions"
+            )
+            answer_bhc = df_bhc.iloc[i]["brief_hospital_course"]
+            answer_di = df_di.iloc[i]["discharge_instructions"]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt_bhc),
+                    references=[Reference(Output(text=answer_bhc), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+            instances.append(
+                Instance(
+                    input=Input(text=prompt_di),
+                    references=[Reference(Output(text=answer_di), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def read_file(self, file_path: str) -> List[str]:
+        with open(file_path, "r") as file:
+            lines = file.readlines()
+        lines = [line.strip() for line in lines]
+        return lines
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="dischargeme",
+            display_name="DischargeMe",
+            short_display_name="DischargeMe",
+            description="DischargeMe is a benchmark designed to evaluate clinical text generation. It "
+            "pairs discharge summaries and radiology reports from MIMIC-IV with generation "
+            "tasks such as writing discharge instructions or summarizing the brief hospital "
+            "course. The benchmark assesses a model's ability to generate patient-facing "
+            "documentation that is complete, empathetic, and clinically accurate [(Xu, "
+            "2024)](https://physionet.org/content/discharge-me/1.3/).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate discharge instructions from hospital notes",
+                when="Upon hospital discharge",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="dischargeme_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/disinformation_scenario.py

@@ -2,8 +2,19 @@ import json
 import os
 from typing import List, Dict, Optional
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 REITERATION_DATA_URL = "https://drive.google.com/uc?export=download&id=1uVJbsgPCHFAvH43I6SVvU3Ayo8dh-y_N"
 WEDGING_DATA_URL = "https://drive.google.com/uc?export=download&id=1kWB3_F4Tobc_oVGC_T-a5DHEh-AB4GTc"
@@ -166,3 +177,23 @@ class DisinformationScenario(Scenario):
             instances = self.create_wedging_instances(data)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.capability == "reiteration":
+            name = "disinformation_reiteration"
+            display_name = "Disinformation (reiteration)"
+        elif self.capability == "wedging":
+            name = "disinformation_wedging"
+            display_name = "Disinformation (wedging)"
+        else:
+            raise Exception(f"Unknown capability {self.capability}")
+        return ScenarioMetadata(
+            name=name,
+            display_name=display_name,
+            description="Scenario from [Buchanan et al. "
+            "(2021)](https://cset.georgetown.edu/publication/truth-lies-and-automation/) "
+            "that tests the ability to generate divisive and wedging content.",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="unknown",
+            main_split="valid",
+        )

helm/benchmark/scenarios/dyck_language_scenario.py

@@ -2,7 +2,18 @@ import numpy as np
 import random
 from typing import List, Tuple
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class DyckLanguageScenario(Scenario):
@@ -225,3 +236,16 @@ class DyckLanguageScenario(Scenario):
             not_allowed=train_inputs,
         )
         return train_instances + test_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="dyck_language",
+            display_name="Dyck",
+            description="Scenario testing hierarchical reasoning through the Dyck formal languages "
+            "[(Suzgun et al., 2019)](https://aclanthology.org/W19-3905/).",
+            taxonomy=TaxonomyInfo(
+                task="next-word prediction", what="Dyck formal language", when="n/a", who="n/a", language="synthetic"
+            ),
+            main_metric="exact_match_indicator",
+            main_split="test",
+        )

helm/benchmark/scenarios/echr_judgment_classification_scenario.py

@@ -0,0 +1,113 @@
+from typing import List, Optional
+import json
+import os
+import re
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+
+
+class EchrJudgeScenario(Scenario):
+    """The "Binary Violation" Classification task from the paper Neural Legal Judgment Prediction in English [(Chalkidis et al., 2019)](https://arxiv.org/pdf/1906.02059.pdf).
+
+    The task is to analyze the description of a legal case from the European Court of Human Rights (ECHR),
+    and classify it as positive if any human rights article or protocol has been violated and negative otherwise.
+
+    The case text can be very long, which sometimes results in incorrect model output
+    when using zero-shot predictions in many cases.
+    Therefore, have added two trivial cases to the instructions part.
+
+    Example Prompt:
+        Is the following case a violation of human rights?  (Instructions)
+
+        Case: Human rights have not been violated.          (Trivial No case in instructions)
+        Answer: No
+
+        Case: Human rights have been violated.              (Trivial Yes case in instructions)
+        Answer: Yes
+
+        Case: <TEXT>                                        (In-context examples, if possible)
+        Answer: <Label>                                     (Label is correct answer, Yes or No)
+
+        ...
+        Case: <TEXT>                                        (Target input text)
+        Answer: <Output>                                    (Output ::= Yes | No)
+    """  # noqa: E501
+
+    # Names of the tasks we support
+    name = "echr_judgment_classification"
+    description = 'The "Binary Violation" Classification task from the paper Neural Legal Judgment Prediction in English [(Chalkidis et al., 2019)](https://arxiv.org/pdf/1906.02059.pdf).'  # noqa: E501
+    tags = ["classification", "judgement", "legal"]
+
+    # Dataset file name
+    _DATASET_URL = "https://archive.org/download/ECHR-ACL2019/ECHR_Dataset.zip"
+
+    # Answer labels
+    YES_ANSWER = "Yes"
+    NO_ANSWER = "No"
+
+    # Prompt constants (used in adapter)
+    PROMPT_INPUT = "Case"
+    PROMPT_OUTPUT = "Answer"
+
+    YES_EX = f"\n\n{PROMPT_INPUT}: Human rights have been violated.\n{PROMPT_OUTPUT}: {YES_ANSWER}"
+    NO_EX = f"\n\n{PROMPT_INPUT}: Human rights have not been violated.\n{PROMPT_OUTPUT}: {NO_ANSWER}"
+    INST_EX = f"{NO_EX}{YES_EX}"
+
+    PROMPT_INST = "Is the following case a violation of human rights?"  # Prompt for instructions
+    PROMPT_INST_WITH_EX = f"{PROMPT_INST}{INST_EX}"  # Prompt for instructions with trivial examples
+
+    def __init__(self, filter_max_length: Optional[int] = None):
+        """
+        Args:
+            filter_max_length: Int indicating maximum length for documents. Documents longer than
+                               train_filter_max_length tokens (using whitespace tokenization)
+                               will be filtered out.
+        """
+        super().__init__()
+        self.filter_max_length = filter_max_length
+
+    def count_words(self, text: str) -> int:
+        """Returns the number of words in the text"""
+        return len(re.split(r"\W+", text))
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(data_dir)
+        file_name = self._DATASET_URL.split("/")[-1]
+        download_directory_path = os.path.join(data_dir, self._DATASET_URL)
+        ensure_file_downloaded(
+            source_url=self._DATASET_URL,
+            target_path=download_directory_path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+
+        source_split_to_helm_split = {"EN_train": TRAIN_SPLIT, "EN_dev": VALID_SPLIT, "EN_test": TEST_SPLIT}
+
+        instances: List[Instance] = []
+        for source_split, helm_split in source_split_to_helm_split.items():
+            target_data_dir = os.path.join(download_directory_path, source_split)
+            for file_name in os.listdir(target_data_dir):
+                if not file_name.endswith(".json"):
+                    continue
+                file_path = os.path.join(target_data_dir, file_name)
+                with open(file_path, "r") as f:
+                    raw_data = json.load(f)
+                input_text = " ".join(raw_data["TEXT"])
+                if self.filter_max_length is not None and self.count_words(input_text) > self.filter_max_length:
+                    continue
+                answer = self.YES_ANSWER if len(raw_data["VIOLATED_ARTICLES"]) > 0 else self.NO_ANSWER
+                correct_reference = Reference(Output(text=answer), tags=[CORRECT_TAG])
+                instances.append(Instance(input=Input(input_text), references=[correct_reference], split=helm_split))
+        return instances

helm/benchmark/scenarios/ehr_sql_scenario.py

@@ -0,0 +1,137 @@
+import os
+import json
+from typing import List
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    Input,
+    Instance,
+    Output,
+    Reference,
+    Scenario,
+)
+from helm.common.general import ensure_directory_exists, ensure_file_downloaded
+
+
+class EhrSqlScenario(Scenario):
+    """
+    Scenario for the EHR SQL dataset.
+
+    - Downloads and sets up the EHR SQL dataset.
+    - Ensures the `eicu.sqlite` database is available for evaluation.
+    - Extracts schema from `eicu.sql` to pass it to the LLM.
+    - Includes `value` field as alternative ground truth result.
+    """
+
+    BASE_URL: str = (
+        "https://raw.githubusercontent.com/glee4810/EHRSQL/e172ec8e61391ecae2d872c8d0ba02a622222f54/dataset/ehrsql/eicu"
+    )
+    DB_URL: str = (
+        "https://github.com/raulista1997/benchmarkdata/raw/c4c252443fa9c52afb6960f53e51be278639bea2/eicu.sqlite"
+    )
+    SQL_SCHEMA_URL: str = (
+        "https://raw.githubusercontent.com/glee4810/EHRSQL/"
+        "e172ec8e61391ecae2d872c8d0ba02a622222f54/dataset/ehrsql/eicu/eicu.sql"
+    )
+
+    name = "ehr_sql"
+    description = (
+        "EHRSQL is a benchmark designed to evaluate models on generating structured queries"
+        "for clinical research. Each example includes a natural language question and a database"
+        "schema, and the task is to produce an SQL query that would return the correct result"
+        "for a biomedical research objective. This benchmark assesses a model's understanding"
+        "of medical terminology, data structures, and query construction."
+    )
+    tags = ["sql", "medical", "reasoning"]
+
+    def setup_database(self, output_path: str) -> str:
+        """Ensure the SQLite database is downloaded and accessible."""
+        ensure_directory_exists(output_path)
+        db_path = os.path.join(output_path, "eicu.sqlite")
+        ensure_file_downloaded(self.DB_URL, db_path)
+        return db_path
+
+    def download_sql_schema(self, output_path: str) -> str:
+        """Download the SQL schema file containing table definitions."""
+        ensure_directory_exists(output_path)
+        schema_path = os.path.join(output_path, "eicu.sql")
+        ensure_file_downloaded(self.SQL_SCHEMA_URL, schema_path)
+        return schema_path
+
+    def extract_schema(self, schema_path: str) -> str:
+        """Extracts and returns `CREATE TABLE` statements from the SQL schema file."""
+        schema_text = []
+        with open(schema_path, "r", encoding="utf-8") as f:
+            lines = f.readlines()
+
+        inside_create_table = False
+        current_table_def = []
+
+        for line in lines:
+            line = line.strip()
+
+            if line.startswith("CREATE TABLE"):
+                inside_create_table = True
+                current_table_def = [line]
+            elif inside_create_table:
+                current_table_def.append(line)
+                if line.endswith(";"):  # End of table definition
+                    inside_create_table = False
+                    schema_text.append("\n".join(current_table_def))
+
+        return "\n\n".join(schema_text)
+
+    def download_json(self, split_name: str, output_path: str) -> str:
+        """Download a specific JSON file for the given split (train, valid, test)."""
+        json_path = os.path.join(output_path, f"ehrsql_{split_name}.json")
+        json_url = f"{self.BASE_URL}/{split_name}.json"
+        ensure_file_downloaded(json_url, json_path)
+        return json_path
+
+    def process_json(self, json_path: str, schema_prompt: str, split: str) -> List[Instance]:
+        """Read and process the JSON file to generate HELM instances."""
+        instances: List[Instance] = []
+        with open(json_path, "r", encoding="utf-8") as f:
+            raw_data = json.load(f)
+            for entry in raw_data:
+                if "question" in entry and "query" in entry:
+                    is_impossible = entry.get("is_impossible", False)
+                    input_text = f"-- Database Schema:\n{schema_prompt}\n\n-- \
+                    Question:\n{entry['question']}\n\n"
+
+                    instance = Instance(
+                        input=Input(text=input_text),
+                        references=[Reference(Output(text=entry["query"]), tags=[CORRECT_TAG])],
+                        extra_data={
+                            "db_path": "eicu.sqlite",
+                            "value": entry.get("value", {}),
+                            "is_impossible": is_impossible,
+                        },
+                        split=split,
+                    )
+
+                    instances.append(instance)
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Download and process the dataset to generate HELM instances."""
+        ensure_directory_exists(output_path)
+
+        # Ensure SQLite database is downloaded
+        self.setup_database(output_path)
+
+        # Ensure schema file is downloaded and extract schema
+        schema_path = self.download_sql_schema(output_path)
+        schema_prompt = self.extract_schema(schema_path)
+
+        # Process dataset
+        splits = {"train": TRAIN_SPLIT, "valid": TEST_SPLIT, "test": TEST_SPLIT}
+        instances: List[Instance] = []
+
+        for split_name, helm_split in splits.items():
+            json_path = self.download_json(split_name, output_path)
+            instances.extend(self.process_json(json_path, schema_prompt, helm_split))
+
+        return instances