crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/scenarios/cti_to_mitre_scenario.py

@@ -0,0 +1,240 @@
+import os
+import json
+from random import Random
+from typing import Any, List, Dict
+
+import pandas as pd
+from pandas import DataFrame
+
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class CtiToMitreScenario(Scenario):
+    """
+    Original Task:
+    - The original task is to classify the description of the situation regarding the system
+      into the security threats in that situation.
+    - The classification categories are the approximately 200 categories of attack techniques
+      in the enterprise as defined by MITRE ATT&CK v10.1.
+
+    Implemented Task:
+    - Since classification into so many classes is difficult to handle in a generative language model
+      such as GPT itself, we implement this task as a multiple-choice task.
+    - Each choice is the name of the attack technique category into which the description is classified.
+    - The number of options is determined by the parameter (num_options).
+        - The minimum number of options is 2 and the maximum is 199, the number of all categories of
+          attack methods defined in MITRE ATT&CK v10.1.
+    - From the 199 choices, num_options choices, including the correct answer and a default case,
+      are randomly selected and used.
+        - If num_options is not specified, all 199 category names will be used as choices.
+
+    Data:
+    - dataset.csv
+        - Target dataset
+        - https://github.com/dessertlab/cti-to-mitre-with-nlp/raw/a8cacf3185d098c686e0d88768a619a03a4d76d1/data/dataset.csv
+        - This data is of the form [sentence, label_tec, label_subtec, tec_name]
+            - sentence: the description
+            - label_tec: label for attack technique category
+            - label_subtec: label for attack technique subcategory
+            - tec_name : name(simple description) for attack technique subcategory
+                - Note: we need to extract name for attack technique category
+                        from enterprise-attack.json
+
+    - enterprise-attack.json
+        - https://github.com/mitre/cti/archive/refs/tags/ATT&CK-v10.1.zip
+            - /mitre_v10/enterprise-attack/enterprise-attack.json
+        - This data contains relation from attack technique name to attack technique label
+            - we can extract attack technique category name for label_tec using this json data.
+
+
+    Prompt: (k is specified by num_options)
+        -----------------------
+        Answer the possible security attacks in each of the following situations from each of the options below.
+                                        [instruction]
+
+        Situation: <description>        [in context examples]
+        A. <attack_category_name_1>
+        B. <attack_category_name_2>
+        ...
+        Y. <attack_category_name_(k-1)>
+        Z. Others
+        Answer: <correct_answer>
+
+        ... (Examples are output as long as the length allows) ...
+
+        Situation: <target_description>  [target question]
+        A. <attack_category_name_t1>
+        B. <attack_category_name_t2>
+        ...
+        Y. <attack_category_name_t(k-1)>
+        Z. Others
+        Answer:
+        -----------------------
+
+    Example of prompt (num_options = 5)
+        -----------------------
+        Answer the possible security attacks in each of the following situations from each of the options below.
+
+        Situation: ZxShell can launch a reverse command shell.
+        A. Command and Scripting Interpreter
+        B. System Shutdown/Reboot
+        C. Exfiltration Over C2 Channel
+        D. Direct Volume Access
+        E. Others
+        Answer: A
+
+        ....(Omitted)...
+
+        Situation: APC injection is a method of executing arbitrary code in the address space.
+        A. Event Triggered Execution
+        B. Process Injection
+        C. Non-Application Layer Protocol
+        D. Escape to Host
+        E. Others
+        Answer: B
+
+        Situation: Timestomping may be used along with file name Masquerading to hide malware and tools.
+        A. Search Victim-Owned Websites
+        B. Internal Spearphishing
+        C. Application Layer Protocol
+        D. Indicator Removal on Host
+        E. Others
+        Answer:
+        -----------------------
+    """  # noqa: E501
+
+    # Names of the tasks we support
+    name = "cti_to_mitre"
+    description = "Classification of security attack opportunities on system"
+    tags = ["classification", "cyber_security"]
+
+    # Constant for splitting target data into train and test data.
+    train_ratio = 0.7
+
+    # Constant for default number of options. # of (MITRE ATT&CK attack categories) is 199 in ATT&CK-v10.1.zip
+    MAX_NUM_OPTIONS = 199
+
+    # Constant: the description for Others option
+    OTHERS_OPTION = "Others"
+
+    CTI_URL = "https://github.com/dessertlab/cti-to-mitre-with-nlp/raw/a8cacf3185d098c686e0d88768a619a03a4d76d1/data/dataset.csv"  # noqa: E501
+    MITRE_URL = "https://github.com/mitre/cti/raw/refs/tags/ATT&CK-v10.1/enterprise-attack/enterprise-attack.json"
+
+    def __init__(self, num_options: int = MAX_NUM_OPTIONS, seed: int = 42) -> None:
+        """
+        num_options: int, number of choices in multiple-choice task
+        seed: int, seed for random module. The seed is set to random if specified
+        """
+        super().__init__()
+        self.num_options = min(num_options, CtiToMitreScenario.MAX_NUM_OPTIONS)
+        self.random_seed = seed
+        self.random = Random(seed)
+
+    @staticmethod
+    def make_label_category_name_dict(jdata: Dict[str, Any]) -> Dict[str, str]:
+        """
+        This makes mapping from label_tec (attack technique category label) to tec_category_name
+        (attack technique category name)
+        - jdata is json object for enterprise_attack.json
+        """
+
+        category_id_to_name: Dict[str, str] = {}
+        attacks = [
+            o for o in jdata["objects"] if o["type"] == "attack-pattern" and not o.get("x_mitre_is_subtechnique", True)
+        ]
+        for attack in attacks:
+            ids = [ref["external_id"] for ref in attack["external_references"] if ref["source_name"] == "mitre-attack"]
+            assert len(ids) == 1
+            id = ids[0]
+            category_id_to_name[id] = attack["name"]
+        return category_id_to_name
+
+    def get_references(self, num_references: int, correct_cname: str, cnames: List[str]) -> List[Reference]:
+        """
+        Randomly select k tec_category_names (attack technique category names) as choices.
+        However, choose not to include "excluded",
+        and if k is less than the total number of possible choices, add a default case.
+        - k : number of choices
+        - correct_cname : correct attack technique category names
+        - cnames : list containing all attack technique category names
+        """
+        assert num_references >= 2, "Need at least 2 references for the correct choice and 'Others'"
+        num_incorrect_cname_samples = num_references - 2
+        assert num_references <= len(
+            cnames
+        ), f"Cannot have more references than the number of categories, which is {len(cnames)}"
+        incorrect_cnames = [cname for cname in cnames if cname != correct_cname]
+        incorrect_cname_samples = self.random.sample(
+            incorrect_cnames, min(len(incorrect_cnames), num_incorrect_cname_samples)
+        )
+        references = [Reference(Output(text=cname), tags=[]) for cname in incorrect_cname_samples]
+        references.append(Reference(Output(text=correct_cname), tags=[CORRECT_TAG]))
+        self.random.shuffle(references)
+        if num_references <= len(cnames):
+            references.append(Reference(Output(text=CtiToMitreScenario.OTHERS_OPTION), tags=[]))
+        return references
+
+    def create_multiple_choice_instances(
+        self, df: DataFrame, split: str, label_cname: Dict[str, str]
+    ) -> List[Instance]:
+        """Create a list of instances corresponding to the multiple choice task"""
+        instances = []
+        for idx in df.index:
+            linedata = df.loc[idx]
+            sentence = linedata["sentence"]
+            label_tec = linedata["label_tec"]
+            correct_cname = label_cname[label_tec]
+            all_cnames = [cname for cname in label_cname.values()]
+            references = self.get_references(self.num_options, correct_cname, all_cnames)
+            input = Input(text=sentence)
+            instance = Instance(input, references, split=split)
+            instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(data_dir)
+
+        dataset_path = os.path.join(data_dir, "dataset.csv")
+        ensure_file_downloaded(
+            source_url="https://github.com/dessertlab/cti-to-mitre-with-nlp/raw/a8cacf3185d098c686e0d88768a619a03a4d76d1/data/dataset.csv",  # noqa: E501
+            target_path=dataset_path,
+        )
+
+        labels_path = os.path.join(data_dir, "enterprise-attack.json")
+        ensure_file_downloaded(
+            source_url="https://github.com/mitre/cti/raw/refs/tags/ATT&CK-v10.1/enterprise-attack/enterprise-attack.json",  # noqa: E501
+            target_path=labels_path,
+        )
+
+        # load dataset
+        all_df = pd.read_csv(dataset_path)
+
+        # split all_df into train and test data frames
+        train_df = all_df.sample(frac=CtiToMitreScenario.train_ratio, random_state=self.random_seed)
+        test_df = all_df.drop(train_df.index).sample(frac=1, random_state=self.random_seed)
+
+        # load labels
+        with open(labels_path) as f:
+            jdata = json.load(f)
+
+        # make mapping from label_tec to tec_category_name
+        label_cname = self.make_label_category_name_dict(jdata)
+
+        # create instances from each dataset
+        instances_train = self.create_multiple_choice_instances(train_df, TRAIN_SPLIT, label_cname)
+        instances_test = self.create_multiple_choice_instances(test_df, TEST_SPLIT, label_cname)
+
+        # return all instances
+        all_instances = instances_train + instances_test
+        return all_instances

helm/benchmark/scenarios/custom_mcqa_scenario.py

@@ -2,7 +2,7 @@ import os
 from typing import List
 
 from helm.common.codec import from_jsonl
-from .scenario import Scenario, Instance
+from helm.benchmark.scenarios.scenario import Scenario, Instance
 
 
 class CustomMCQAScenario(Scenario):

helm/benchmark/scenarios/czech_bank_qa_scenario.py

@@ -0,0 +1,130 @@
+import datasets
+import os
+from typing import List
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class CzechBankQAScenario(Scenario):
+    INSTRUCTIONS = """Given a SQLite database schema and the following instructions, generate a SQLite query that corresponds to the instructions. Answer with only the query.
+
+Database schema:
+CREATE TABLE "account" (
+  "account_id" integer NOT NULL DEFAULT '0'
+,  "district_id" integer NOT NULL DEFAULT '0'
+,  "frequency" varchar(18) NOT NULL
+,  "date" date NOT NULL
+,  PRIMARY KEY ("account_id")
+,  CONSTRAINT "account_ibfk_1" FOREIGN KEY ("district_id") REFERENCES "district" ("district_id")
+);
+CREATE TABLE "card" (
+  "card_id" integer NOT NULL DEFAULT '0'
+,  "disp_id" integer NOT NULL
+,  "type" varchar(7) NOT NULL
+,  "issued" date NOT NULL
+,  PRIMARY KEY ("card_id")
+,  CONSTRAINT "card_ibfk_1" FOREIGN KEY ("disp_id") REFERENCES "disp" ("disp_id")
+);
+CREATE TABLE "client" (
+  "client_id" integer NOT NULL
+,  "gender" varchar(1) NOT NULL
+,  "birth_date" date NOT NULL
+,  "district_id" integer NOT NULL
+,  PRIMARY KEY ("client_id")
+,  CONSTRAINT "client_ibfk_1" FOREIGN KEY ("district_id") REFERENCES "district" ("district_id")
+);
+CREATE TABLE "disp" (
+  "disp_id" integer NOT NULL
+,  "client_id" integer NOT NULL
+,  "account_id" integer NOT NULL
+,  "type" varchar(9) NOT NULL
+,  PRIMARY KEY ("disp_id")
+,  CONSTRAINT "disp_ibfk_1" FOREIGN KEY ("account_id") REFERENCES "account" ("account_id")
+,  CONSTRAINT "disp_ibfk_2" FOREIGN KEY ("client_id") REFERENCES "client" ("client_id")
+);
+CREATE TABLE "district" (
+  "district_id" integer NOT NULL DEFAULT '0'
+,  "A2" varchar(19) NOT NULL
+,  "A3" varchar(15) NOT NULL
+,  "A4" integer NOT NULL
+,  "A5" integer NOT NULL
+,  "A6" integer NOT NULL
+,  "A7" integer NOT NULL
+,  "A8" integer NOT NULL
+,  "A9" integer NOT NULL
+,  "A10" decimal(4,1) NOT NULL
+,  "A11" integer NOT NULL
+,  "A12" decimal(4,1) DEFAULT NULL
+,  "A13" decimal(3,2) NOT NULL
+,  "A14" integer NOT NULL
+,  "A15" integer DEFAULT NULL
+,  "A16" integer NOT NULL
+,  PRIMARY KEY ("district_id")
+);
+CREATE TABLE "loan" (
+  "loan_id" integer NOT NULL DEFAULT '0'
+,  "account_id" integer NOT NULL
+,  "date" date NOT NULL
+,  "amount" integer NOT NULL
+,  "duration" integer NOT NULL
+,  "payments" decimal(6,2) NOT NULL
+,  "status" varchar(1) NOT NULL
+,  PRIMARY KEY ("loan_id")
+,  CONSTRAINT "loan_ibfk_1" FOREIGN KEY ("account_id") REFERENCES "account" ("account_id")
+);
+CREATE TABLE "order" (
+  "order_id" integer NOT NULL DEFAULT '0'
+,  "account_id" integer NOT NULL
+,  "bank_to" varchar(2) NOT NULL
+,  "account_to" integer NOT NULL
+,  "amount" decimal(6,1) NOT NULL
+,  "k_symbol" varchar(8) NOT NULL
+,  PRIMARY KEY ("order_id")
+,  CONSTRAINT "order_ibfk_1" FOREIGN KEY ("account_id") REFERENCES "account" ("account_id")
+);
+CREATE TABLE "trans" (
+  "trans_id" integer NOT NULL DEFAULT '0'
+,  "account_id" integer NOT NULL DEFAULT '0'
+,  "date" date NOT NULL
+,  "type" varchar(6) NOT NULL
+,  "operation" varchar(14) DEFAULT NULL
+,  "amount" integer NOT NULL
+,  "balance" integer NOT NULL
+,  "k_symbol" varchar(11) DEFAULT NULL
+,  "bank" varchar(2) DEFAULT NULL
+,  "account" integer  DEFAULT NULL
+,  PRIMARY KEY ("trans_id")
+,  CONSTRAINT "trans_ibfk_1" FOREIGN KEY ("account_id") REFERENCES "account" ("account_id")
+);"""  # noqa: E501
+
+    """CzechBankQA"""
+    name = "czech_bank_qa"
+    description = "This is a list of SQL queries for a text-to-SQL task over the Czech Bank 1999 dataset."
+    tags = ["text_to_sql"]
+
+    def __init__(self, config_name: str):
+        super().__init__()
+        self.config_name = config_name
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "yifanmai/czech_bank_qa", name=self.config_name, split="test", cache_dir=cache_dir
+        )
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=row["description"])
+            references = [Reference(output=Output(text=row["sql_query"]), tags=[CORRECT_TAG])]
+            instance = Instance(input=input, references=references, split=TEST_SPLIT)
+            instances.append(instance)
+        return instances

helm/benchmark/scenarios/decodingtrust_adv_demonstration_scenario.py

@@ -3,7 +3,7 @@ import json
 from typing import List
 from helm.common.general import ensure_file_downloaded
 from helm.common.general import ensure_directory_exists
-from .scenario import Scenario, Instance, Input, Reference, CORRECT_TAG, Output, VALID_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Reference, CORRECT_TAG, Output, VALID_SPLIT
 
 TASKS = {
     "counterfactual": ["snli_premise", "snli_hypothesis"]

helm/benchmark/scenarios/decodingtrust_privacy_scenario.py

@@ -5,7 +5,7 @@ import random
 from tqdm import tqdm
 from typing import List, Dict, Optional
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Input, Reference, Output, TEST_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Reference, Output, TEST_SPLIT
 
 option_keys = ["pii", "privacy_understanding", "enron_email_extraction"]
 

helm/benchmark/scenarios/decodingtrust_stereotype_bias_scenario.py

@@ -1,7 +1,7 @@
 import json
 import os
 from typing import List, Dict
-from .scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import Reference, Output
 

helm/benchmark/scenarios/decodingtrust_toxicity_prompts_scenario.py

@@ -4,7 +4,7 @@ import random
 from typing import List, Dict
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
 
 
 DATA_REPO_HASH = "38972f6ccbf376a8d0660babafb4d2b3b9cca3f4"

helm/benchmark/scenarios/dialogue_scenarios.py

@@ -6,7 +6,18 @@ from typing import Dict, List, Optional
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.runner import get_benchmark_output_path
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 @dataclass(frozen=True)
@@ -138,5 +149,5 @@ class EmpatheticDialoguesScenario(Scenario):
 
 if __name__ == "__main__":
     scenario = EmpatheticDialoguesScenario()
-    instances = scenario.get_instances("./benchmark_output/scenarios/empatheticdialogues")
+    instances = scenario.get_instances(os.path.join(get_benchmark_output_path(), "scenarios/empatheticdialogues"))
     print(instances[100])

helm/benchmark/scenarios/dischargeme_scenario.py

@@ -0,0 +1,157 @@
+from typing import List
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+import pandas as pd
+
+
+def file_preprocessing(data_path: str, task_objective: str) -> pd.DataFrame:
+    """
+    Preprocess the data files to create a DataFrame with the necessary columns.
+    task_objective: 'brief_hospital_course' or 'discharge_instructions'
+    Use command to download: wget -r -N -c -np --user {PHYSIONET_USERNAME} \
+    --ask-password https://physionet.org/files/discharge-me/1.3/
+    data_path is directory that contains the downloaded files: '{base_dir}/physionet.org/'
+    """
+    # Load the first CSV file
+    df_diagnosis = pd.read_csv(
+        f"{data_path}/files/discharge-me/1.3/test_phase_1/diagnosis.csv.gz", compression="gzip", keep_default_na=False
+    )
+    df_discharge = pd.read_csv(
+        f"{data_path}/files/discharge-me/1.3/test_phase_1/discharge.csv.gz", compression="gzip", keep_default_na=False
+    )
+    df_target = pd.read_csv(
+        f"{data_path}/files/discharge-me/1.3/test_phase_1/discharge_target.csv.gz",
+        compression="gzip",
+        keep_default_na=False,
+    )
+    df_radiology = pd.read_csv(
+        f"{data_path}/files/discharge-me/1.3/test_phase_1/radiology.csv.gz", compression="gzip", keep_default_na=False
+    )
+    df_ed = pd.read_csv(
+        f"{data_path}/files/discharge-me/1.3/test_phase_1/edstays.csv.gz", compression="gzip", keep_default_na=False
+    )
+    df_triage = pd.read_csv(
+        f"{data_path}/files/discharge-me/1.3/test_phase_1/triage.csv.gz", compression="gzip", keep_default_na=False
+    )
+    df_diagnosis_triage = pd.merge(
+        df_diagnosis, df_triage, on="subject_id", how="inner", suffixes=("_df_diagnosis", "_df_triage")
+    )
+    df_diagnosis_triage_discharge = pd.merge(
+        df_diagnosis_triage, df_discharge, on="subject_id", how="inner", suffixes=("", "_df_discharge")
+    )
+    df_diagnosis_triage_discharge_radiology = pd.merge(
+        df_diagnosis_triage_discharge, df_radiology, on="hadm_id", how="inner", suffixes=("", "_df_radiology")
+    )
+
+    df_features = pd.merge(
+        df_diagnosis_triage_discharge_radiology, df_ed, on="hadm_id", how="inner", suffixes=("", "_df_ed")
+    )
+
+    # Reduce the DataFrame to remove duplicate hadm_id
+    df_features_reduced = df_features.drop_duplicates(subset="hadm_id")
+    columns_to_keep = ["text", "text_df_radiology", "hadm_id"]
+    df_input = df_features_reduced[columns_to_keep]
+    final_df = pd.merge(df_input, df_target, on="hadm_id", how="inner")
+
+    def remove_substring(string, substring):
+        return string.replace(substring, "")
+
+    final_df["text"] = final_df.apply(lambda row: remove_substring(row["text"], row[task_objective]), axis=1)
+    return final_df
+
+
+def create_prompt(text: str, text_df_radiology: str, task_objective: str) -> str:
+    """
+    Create the prompt for the instance.
+    """
+    prompt = f"Generate the {task_objective} from the following patient discharge text and radiology report text.\
+    \n\nDischarge Text:\n{text}\n\nRadiology Report:\n{text_df_radiology}\n\n{task_objective}:\n"
+    return prompt
+
+
+class DischargeMeScenario(Scenario):
+    """
+    DischargeMe is a discharge instruction generation dataset and brief hospital course generation \
+    dataset collected from MIMIC-IV data.
+    In this scenario, we only consider the discharge text as well as the radiology report text.
+    We are using the phase I test set which is composed of 14,702 hospital admission instances.
+
+    The splits are provided by the dataset itself.
+
+    TASKS = {discharge instruction, brief hospital course}
+    Sample Synthetic Prompt:
+        Generate the {TASK} from the following patient discharge text and radiology report text.
+
+        Discharge Text:
+        Name: {Patient Name} Unit No: {Unit Number} Date of Birth: {DOB} Date of Admission:
+        {DOA} Date of Discharge: {DOD}
+        Chief Complaint: {Chief Complaint} History of Present Illness: {HPI} Past Medical History: {PMH}
+        Medications on Admission: {Medications} Allergies: {Allergies} Physical Exam: {Physical Exam}
+        Discharge Diagnosis: {Discharge Diagnosis}
+
+        Radiology Report:
+        {Radiology Report}
+
+        {TASK}:
+    @inproceedings{Xu_2024,
+        title={ Discharge me: Bionlp acl’24 shared task on streamlining discharge documentation.},
+        url={https://doi.org/10.13026/4a0k-4360},
+        DOI={10.13026/27pt-1259},
+        booktitle={ Proceedings of the 23rd Workshop on Biomedical Natural Language Processing (BioNLP) at ACL 2024},
+        publisher={Association for Computational Linguistics},
+        author={Xu, Justin and Delbrouck, Jean-Benoit and Johnston, Andrew and Blankemeier, Louis and Langlotz, Curtis},
+        year={2024}
+    }
+    """
+
+    name = "dischargeme"
+    description = "DischargeMe is a discharge instruction generation dataset and brief hospital course generation \
+    dataset collected from MIMIC-IV data, consindering only the discharge text as well as the radiology report text."
+    tags = ["biomedical"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/share/pi/nigam/data/physionet.org"
+        ensure_directory_exists(data_path)
+        instances: List[Instance] = []
+        df_bhc = file_preprocessing(data_path, "brief_hospital_course")
+        df_di = file_preprocessing(data_path, "discharge_instructions")
+
+        for i in range(df_bhc.shape[0]):
+            prompt_bhc = create_prompt(
+                df_bhc.iloc[i]["text"], df_bhc.iloc[i]["text_df_radiology"], "Brief Hospital Course"
+            )
+            prompt_di = create_prompt(
+                df_di.iloc[i]["text"], df_di.iloc[i]["text_df_radiology"], "Discharge Instructions"
+            )
+            answer_bhc = df_bhc.iloc[i]["brief_hospital_course"]
+            answer_di = df_di.iloc[i]["discharge_instructions"]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt_bhc),
+                    references=[Reference(Output(text=answer_bhc), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+            instances.append(
+                Instance(
+                    input=Input(text=prompt_di),
+                    references=[Reference(Output(text=answer_di), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def read_file(self, file_path: str) -> List[str]:
+        with open(file_path, "r") as file:
+            lines = file.readlines()
+        lines = [line.strip() for line in lines]
+        return lines

helm/benchmark/scenarios/disinformation_scenario.py

@@ -3,7 +3,16 @@ import os
 from typing import List, Dict, Optional
 
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 REITERATION_DATA_URL = "https://drive.google.com/uc?export=download&id=1uVJbsgPCHFAvH43I6SVvU3Ayo8dh-y_N"
 WEDGING_DATA_URL = "https://drive.google.com/uc?export=download&id=1kWB3_F4Tobc_oVGC_T-a5DHEh-AB4GTc"

helm/benchmark/scenarios/dyck_language_scenario.py

@@ -2,7 +2,16 @@ import numpy as np
 import random
 from typing import List, Tuple
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 class DyckLanguageScenario(Scenario):