crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/ruler_qa_scenario_helper.py

@@ -133,7 +133,7 @@ def generate_samples(dataset: str, dataset_path: str, template: str, random_seed
         input_text, answer = generate_input_output(0, num_docs, template=template, random_seed=random_seed, qas=qas, docs=docs)
         # Calculate the number of tokens in the example
         total_tokens = len(_text_to_tokens(input_text + f' {answer}'))
-        print(f'Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate} | Docs: {num_docs}')
+        # print(f'Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate} | Docs: {num_docs}')
         if total_tokens + tokens_to_generate > max_seq_length:
             num_docs -= incremental
             break
@@ -142,7 +142,7 @@ def generate_samples(dataset: str, dataset_path: str, template: str, random_seed
         if num_docs > len(docs):
             num_docs = len(docs)
             break
-    print('Number of documents:', num_docs)
+    # print('Number of documents:', num_docs)
     
     # Generate samples
     for index in tqdm(range(num_samples)):

helm/benchmark/scenarios/ruler_qa_scenarios.py

@@ -72,7 +72,7 @@ Question: {query} Answer:"""  # noqa: E501
 
 class RULERHotpotQAScenario(_RULERQAScenario):
     name = "ruler_hotpotqa"
-    description = "The HotpotQA long-context multi-hop RAG question answering scenario from RULER"
+    description = "RULER HotPotQA is an augmented version of HotPotQA ([Yang et al., 2018](https://arxiv.org/abs/1809.09600)) introduced by [Hsieh et al., 2024](https://arxiv.org/abs/2404.06654) to simulate a multi-hop question answering as a long-context scenario."  # noqa: E501
     tags = ["long_context", "rag"]
 
     def __init__(self, max_num_words: int):
@@ -81,7 +81,7 @@ class RULERHotpotQAScenario(_RULERQAScenario):
 
 class RULERSQuADScenario(_RULERQAScenario):
     name = "ruler_squad"
-    description = "The SQuAD question answering scenario from RULER"
+    description = "RULER SQuAD is an augmented version of SQuAD ([Rajpurkar et al., 2018](https://arxiv.org/abs/1806.03822)) introduced by [Hsieh et al., 2024](https://arxiv.org/abs/2404.06654) to simulate a single-hop question answering as a long-context scenario."  # noqa: E501
     tags = ["long_context", "rag"]
 
     def __init__(self, max_num_words: int):

helm/benchmark/scenarios/seahelm_scenario.py

@@ -1750,7 +1750,7 @@ class LINDSEAPragmaticsPresuppositionsScenario(Scenario):
                 text_noun = self.prompt_components["text_noun"]
                 instruction = self.prompt_components["single_instruction"]
 
-                passage = "{question}\{text_noun}: {text}\n{instruction}".format(
+                passage = "{question}\\{text_noun}: {text}\n{instruction}".format(
                     question=question.format(row["question_translated"]),
                     text_noun=text_noun,
                     text=row["text"],
@@ -1898,7 +1898,7 @@ class LINDSEAPragmaticsScalarImplicaturesScenario(Scenario):
                 text_noun = self.prompt_components["text_noun"]
                 instruction = self.prompt_components["single_instruction"]
 
-                passage = "{question}\{text_noun}: {text}\n{instruction}".format(
+                passage = "{question}\\{text_noun}: {text}\n{instruction}".format(
                     question=question.format(row["question_translated"]),
                     text_noun=text_noun,
                     text=row["text"],

helm/benchmark/scenarios/shc_bmt_scenario.py

@@ -11,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
 )
+from helm.common.general import check_file_exists
 
 csv.field_size_limit(sys.maxsize)
 
@@ -23,13 +24,19 @@ class SHCBMTMedScenario(Scenario):
 
     name = "shc_bmt_med"
     description = (
-        "A dataset containing patient notes with associated "
-        "questions and answers related to bone marrow transplantation."
+        "BMT-Status is a benchmark composed of clinical notes and associated binary questions"
+        "related to bone marrow transplant (BMT), hematopoietic stem cell transplant (HSCT),"
+        "or hematopoietic cell transplant (HCT) status. The goal is to determine whether the"
+        "patient received a subsequent transplant based on the provided clinical documentation."
     )
     tags = ["knowledge", "reasoning", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def create_benchmark(self, csv_path) -> Dict[str, str]:
         data = {}
         with open(csv_path, "r") as file:
@@ -39,7 +46,7 @@ class SHCBMTMedScenario(Scenario):
                 context = row["context"]
                 answer = row["label"]
                 prompt = (
-                    f"Provide an answer to the following {question} with the following context: {context} "
+                    f"Provide an answer to the following question: {question} with the following context: {context} "
                     ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
                     "details or response, just a simple A or B response."
                 )
@@ -47,10 +54,9 @@ class SHCBMTMedScenario(Scenario):
         return data
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-BMT-dataset_filtered.csv"
-
+        check_file_exists(self.data_path, msg=f"[SHCBMTMedScenario] Required data file not found: '{self.data_path}'")
         instances: List[Instance] = []
-        benchmark_data = self.create_benchmark(data_path)
+        benchmark_data = self.create_benchmark(self.data_path)
 
         for prompt, answer in benchmark_data.items():
             assert answer in SHCBMTMedScenario.POSSIBLE_ANSWER_CHOICES

helm/benchmark/scenarios/shc_cdi_scenario.py

@@ -11,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
 )
+from helm.common.general import check_file_exists
 
 csv.field_size_limit(sys.maxsize)
 
@@ -24,13 +25,18 @@ class SHCCDIMedScenario(Scenario):
 
     name = "shc_cdi_med"
     description = (
-        "A dataset built from Clinical Document Integrity (CDI) notes, to assess "
-        "the ability to answer verification questions from previous notes."
+        "CDI-QA is a benchmark constructed from Clinical Documentation Integrity (CDI)"
+        "notes. It is used to evaluate a model's ability to verify clinical conditions based on"
+        "documented evidence in patient records."
     )
     tags = ["knowledge", "reasoning", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def create_benchmark(self, csv_path) -> Dict[str, str]:
         data = {}
         with open(csv_path, "r") as file:
@@ -40,7 +46,7 @@ class SHCCDIMedScenario(Scenario):
                 context = row["context"]
                 answer = row["label"]
                 prompt = (
-                    f"Provide an answer to the following {question} with the following context: {context} , "
+                    f"Provide an answer to the following question: {question} with the following context: {context} , "
                     "Answer the question with either 'A' for yes or 'B' for no. Do not provide any "
                     "additional details or response, just a simple A or B response."
                 )
@@ -48,10 +54,9 @@ class SHCCDIMedScenario(Scenario):
         return data
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-CDI-dataset_filtered.csv"
-
+        check_file_exists(self.data_path, msg=f"[SHCCDIMedScenario] Required data file not found: '{self.data_path}'")
         instances: List[Instance] = []
-        benchmark_data = self.create_benchmark(data_path)
+        benchmark_data = self.create_benchmark(self.data_path)
 
         for prompt, answer in benchmark_data.items():
             assert answer in SHCCDIMedScenario.POSSIBLE_ANSWER_CHOICES

helm/benchmark/scenarios/shc_conf_scenario.py

@@ -11,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
 )
+from helm.common.general import check_file_exists
 
 csv.field_size_limit(sys.maxsize)
 
@@ -24,13 +25,19 @@ class SHCCONFMedScenario(Scenario):
 
     name = "shc_conf_med"
     description = (
-        "A dataset of clinical notes from adolescent patients used to identify sensitive "
-        "protected health information that should be restricted from parental access."
+        "MedConfInfo is a benchmark comprising clinical notes from adolescent patients. It is"
+        "used to evaluate whether the content contains sensitive protected health information"
+        "(PHI) that should be restricted from parental access, in accordance with adolescent"
+        "confidentiality policies in clinical care."
     )
     tags = ["knowledge", "reasoning", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def create_benchmark(self, csv_path) -> Dict[str, str]:
         data = {}
         with open(csv_path, "r") as file:
@@ -40,7 +47,7 @@ class SHCCONFMedScenario(Scenario):
                 context = row["context"]
                 answer = row["label"]
                 prompt = (
-                    f"Provide an answer to the following {question} with the following context: {context} "
+                    f"Provide an answer to the following question: {question} with the following context: {context} "
                     ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
                     "details or response, just a simple A or B response."
                 )
@@ -48,10 +55,9 @@ class SHCCONFMedScenario(Scenario):
         return data
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-CONF-dataset_filtered.csv"
-
+        check_file_exists(self.data_path, msg=f"[SHCCONFMedScenario] Required data file not found: '{self.data_path}'")
         instances: List[Instance] = []
-        benchmark_data = self.create_benchmark(data_path)
+        benchmark_data = self.create_benchmark(self.data_path)
 
         for prompt, answer in benchmark_data.items():
             assert answer in SHCCONFMedScenario.POSSIBLE_ANSWER_CHOICES

helm/benchmark/scenarios/shc_ent_scenario.py

@@ -11,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
 )
+from helm.common.general import check_file_exists
 
 csv.field_size_limit(sys.maxsize)
 
@@ -23,13 +24,18 @@ class SHCENTMedScenario(Scenario):
 
     name = "shc_ent_med"
     description = (
-        "A dataset designed to evaluate performance in "
-        "identifying appropriate patient referrals to Ear, Nose, and Throat specialists."
+        "ENT-Referral is a benchmark designed to evaluate whether a patient's clinical note"
+        "supports a referral to an Ear, Nose, and Throat (ENT) specialist. It helps assess"
+        "models' abilities to make referral decisions based on unstructured clinical text."
     )
     tags = ["knowledge", "reasoning", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B", "C"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def create_benchmark(self, csv_path) -> Dict[str, str]:
         data = {}
         counter = 1
@@ -41,7 +47,7 @@ class SHCENTMedScenario(Scenario):
                     context = row["context"]
                     answer = row["label"]
                     prompt = (
-                        f"{counter} Provide an answer to the following {question} with the following context:"
+                        f"{counter} Provide an answer to the following question: {question} with the following context:"
                         f" {context} , Answer the question with either 'A' for yes, 'B' for no, or 'C' for no mention."
                         " Do not provide any additional details or response, just a simple A, B, or C response."
                     )
@@ -50,10 +56,9 @@ class SHCENTMedScenario(Scenario):
         return data
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-ENT-dataset_filtered.csv"
-
+        check_file_exists(self.data_path, msg=f"[SHCENTMedScenario] Required data file not found: '{self.data_path}'")
         instances: List[Instance] = []
-        benchmark_data = self.create_benchmark(data_path)
+        benchmark_data = self.create_benchmark(self.data_path)
 
         for prompt, answer in benchmark_data.items():
             assert answer in SHCENTMedScenario.POSSIBLE_ANSWER_CHOICES

helm/benchmark/scenarios/shc_gip_scenario.py

@@ -11,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
 )
+from helm.common.general import check_file_exists
 
 csv.field_size_limit(sys.maxsize)
 
@@ -22,11 +23,19 @@ class SHCGIPMedScenario(Scenario):
     """
 
     name = "shc_gip_med"
-    description = "A dataset evaluating performance in identifying appropriate patient referrals to hospice care."
+    description = (
+        "HospiceReferral is a benchmark that evaluates model performance in identifying"
+        "whether patients are eligible for hospice care based on palliative care clinical notes."
+        "The benchmark focuses on end-of-life care referral decisions."
+    )
     tags = ["knowledge", "reasoning", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def create_benchmark(self, csv_path) -> Dict[str, str]:
         data = {}
         with open(csv_path, "r") as file:
@@ -36,7 +45,7 @@ class SHCGIPMedScenario(Scenario):
                 context = row["context"]
                 answer = row["label"]
                 prompt = (
-                    f"Provide an answer to the following {question} with the following context: {context} "
+                    f"Provide an answer to the following question: {question} with the following context: {context} "
                     ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
                     "details or response, just a simple A or B response."
                 )
@@ -44,10 +53,9 @@ class SHCGIPMedScenario(Scenario):
         return data
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-GIP-dataset_filtered.csv"
-
+        check_file_exists(self.data_path, msg=f"[SHCGIPMedScenario] Required data file not found: '{self.data_path}'")
         instances: List[Instance] = []
-        benchmark_data = self.create_benchmark(data_path)
+        benchmark_data = self.create_benchmark(self.data_path)
 
         for prompt, answer in benchmark_data.items():
             assert answer in SHCGIPMedScenario.POSSIBLE_ANSWER_CHOICES

helm/benchmark/scenarios/shc_privacy_scenario.py

@@ -0,0 +1,78 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCPRIVACYMedScenario(Scenario):
+    """
+    This dataset features messages sent generated by an LLM from patient clinical notes data.
+    The scenario evaluates the ability of an LLM to determine if any potentially confidential
+    information about the patient was included. From publication: https://doi.org/10.1001/jamapediatrics.2024.4438
+    """
+
+    name = "shc_privacy_med"
+    description = (
+        "PrivacyDetection is a benchmark composed of patient portal messages submitted by"
+        "patients or caregivers. The task is to determine whether the message contains any"
+        "confidential or privacy-leaking information that should be protected."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    "You are reviewing clinical messages in order to determine if any confidential "
+                    f"information was included. Please answer the following question: {question} with the "
+                    f"following context: {context} , Answer the question with a 'A' for yes or 'B' "
+                    "for no. Do not provide any additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(
+            self.data_path, msg=f"[SHCPRIVACYMedScenario] Required data file not found: '{self.data_path}'"
+        )
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCPRIVACYMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCPRIVACYMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_proxy_scenario.py

@@ -0,0 +1,76 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCPROXYMedScenario(Scenario):
+    """
+    This dataset features messages sent by proxy users and non proxy users, for evaluation of
+    LLM capabilities to determine the sender. From publication: https://doi.org/10.1001/jamapediatrics.2024.4438
+    """
+
+    name = "shc_proxy_med"
+    description = (
+        "ProxySender is a benchmark composed of patient portal messages received by clinicians."
+        "It evaluates whether the message was sent by the patient or by a proxy user (e.g., parent,"
+        "spouse), which is critical for understanding who is communicating with healthcare"
+        "providers."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    "You are reviewing a clinical messages in order to determine if they have been "
+                    f"sent by a proxy user. Please determine the following: {question} with the "
+                    f"following context: {context} , Answer the question with a 'A' for yes or 'B' "
+                    "for no. Do not provide any additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[SHCPROXYMedScenario] Required data file not found: '{self.data_path}'")
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCPROXYMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCPROXYMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_ptbm_scenario.py

@@ -11,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
 )
+from helm.common.general import check_file_exists
 
 csv.field_size_limit(sys.maxsize)
 
@@ -27,14 +28,19 @@ class SHCPTBMMedScenario(Scenario):
 
     name = "shc_ptbm_med"
     description = (
-        "A dataset that classifies whether a clinical note contains a clinician "
-        "recommendation for parent training in behavior management, which is the first-line "
-        "evidence-based treatment for young children with ADHD."
+        "ADHD-Behavior is a benchmark that evaluates a model’s ability to detect whether"
+        "a clinician recommends parent training in behavior management, an evidence-based"
+        "first-line treatment for young children diagnosed with ADHD. Each instance includes"
+        "a clinical note from a pediatric visit and a binary classification task."
     )
     tags = ["knowledge", "reasoning", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def create_benchmark(self, csv_path) -> Dict[str, str]:
         data = {}
         with open(csv_path, "r") as file:
@@ -46,7 +52,7 @@ class SHCPTBMMedScenario(Scenario):
                 prompt = (
                     "You are reviewing a clinical note from health records of children with "
                     "attention deficit hyperactivity disorder (ADHD) and classifying mentions of "
-                    f"behavioral therapy. Provide an answer to the following {question} with the "
+                    f"behavioral therapy. Provide an answer to the following question: {question} with the "
                     f"following context: {context} , Answer the question with a 'A' for yes or 'B' "
                     "for no. Do not provide any additional details or response, just a simple A or B response."
                 )
@@ -54,10 +60,9 @@ class SHCPTBMMedScenario(Scenario):
         return data
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-PTBM-dataset_filtered.csv"
-
+        check_file_exists(self.data_path, msg=f"[SHCPTBMMedScenario] Required data file not found: '{self.data_path}'")
         instances: List[Instance] = []
-        benchmark_data = self.create_benchmark(data_path)
+        benchmark_data = self.create_benchmark(self.data_path)
 
         for prompt, answer in benchmark_data.items():
             assert answer in SHCPTBMMedScenario.POSSIBLE_ANSWER_CHOICES

helm/benchmark/scenarios/shc_sei_scenario.py

@@ -11,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
 )
+from helm.common.general import check_file_exists
 
 csv.field_size_limit(sys.maxsize)
 
@@ -30,14 +31,19 @@ class SHCSEIMedScenario(Scenario):
 
     name = "shc_sei_med"
     description = (
-        "A dataset that classifies whether a clinical note contains documentation "
-        "of side effect monitoring (recording of absence or presence of medication "
-        "side effects), as recommended in clinical practice guidelines."
+        "ADHD-MedEffects is a benchmark designed to evaluate whether clinical notes for"
+        "pediatric ADHD visits document medication side effect monitoring, which is a key recommendation"
+        "in clinical practice guidelines. The dataset supports binary classification"
+        "to detect presence or absence of side effect inquiries (SEI) within notes."
     )
     tags = ["knowledge", "reasoning", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def create_benchmark(self, csv_path) -> Dict[str, str]:
         data = {}
         with open(csv_path, "r") as file:
@@ -59,7 +65,7 @@ class SHCSEIMedScenario(Scenario):
                     "categorized as SEI because they consist of a plan or an explanation about "
                     "side effects without actual side effect monitoring taking place, and "
                     "No Side Effects Inquiry (NSEI): No evidence of side effects monitoring. "
-                    f"Provide an answer to the following {question} with the following context: {context} "
+                    f"Provide an answer to the following question: {question} with the following context: {context} "
                     ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
                     "details or response, just a simple A or B response."
                 )
@@ -67,10 +73,9 @@ class SHCSEIMedScenario(Scenario):
         return data
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-SEI-dataset_filtered.csv"
-
+        check_file_exists(self.data_path, msg=f"[SHCSEIMedScenario] Required data file not found: '{self.data_path}'")
         instances: List[Instance] = []
-        benchmark_data = self.create_benchmark(data_path)
+        benchmark_data = self.create_benchmark(self.data_path)
 
         for prompt, answer in benchmark_data.items():
             assert answer in SHCSEIMedScenario.POSSIBLE_ANSWER_CHOICES

helm/benchmark/scenarios/shc_sequoia_scenario.py

@@ -11,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
 )
+from helm.common.general import check_file_exists
 
 csv.field_size_limit(sys.maxsize)
 
@@ -22,12 +23,18 @@ class SHCSequoiaMedScenario(Scenario):
 
     name = "shc_sequoia_med"
     description = (
-        "A dataset containing manually curated answers to questions regarding patient referrals to the Sequoia clinic."
+        "ClinicReferral is a benchmark that determines patient eligibility for referral to the"
+        "Sequoia Clinic based on information from palliative care notes. The dataset provides"
+        "curated decisions on referral appropriateness to assist in automating clinic workflows."
     )
     tags = ["knowledge", "reasoning", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def create_benchmark(self, csv_path) -> Dict[str, str]:
         data = {}
         counter = 1
@@ -38,7 +45,7 @@ class SHCSequoiaMedScenario(Scenario):
                 context = row["context"]
                 answer = row["label"]
                 prompt = (
-                    f" {counter} Provide an answer to the following {question} with the following context:"
+                    f" {counter} Provide an answer to the following question: {question} with the following context:"
                     f" {context} , Answer the question with a 'A' for yes or 'B' for no. Do not provide any "
                     "additional details or response, just a simple A or B response."
                 )
@@ -47,10 +54,11 @@ class SHCSequoiaMedScenario(Scenario):
         return data
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/dbfs/mnt/azure_adbfs/Files/medhelm/medhelm-sequoia-dataset_filtered.csv"
-
+        check_file_exists(
+            self.data_path, msg=f"[SHCSequoiaMedScenario] Required data file not found: '{self.data_path}'"
+        )
         instances: List[Instance] = []
-        benchmark_data = self.create_benchmark(data_path)
+        benchmark_data = self.create_benchmark(self.data_path)
 
         for prompt, answer in benchmark_data.items():
             assert answer in SHCSequoiaMedScenario.POSSIBLE_ANSWER_CHOICES