crfm-helm 0.5.5__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/run_specs/medhelm_run_specs.py

@@ -37,6 +37,7 @@ def get_medcalc_bench_spec() -> RunSpec:
         output_noun="Answer only the requested quantity without units. No explanation needed",
         max_tokens=10,
         max_train_instances=0,
+        stop_sequences=[],
     )
 
     metric_specs = [
@@ -56,9 +57,13 @@ def get_medcalc_bench_spec() -> RunSpec:
 
 
 @run_spec_function("clear")
-def get_clear_spec(condition: str) -> RunSpec:
+def get_clear_spec(condition: str, data_path: str) -> RunSpec:
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.clear_scenario.CLEARScenario", args={"condition": condition}
+        class_name="helm.benchmark.scenarios.clear_scenario.CLEARScenario",
+        args={
+            "condition": condition,
+            "data_path": data_path,
+        },
     )
 
     condition_display = condition.replace("_", " ")
@@ -157,6 +162,7 @@ def get_medec_run_spec() -> RunSpec:
         output_noun="Answer",
         max_tokens=256,
         max_train_instances=0,
+        stop_sequences=[],
     )
 
     # Define the metrics
@@ -178,10 +184,14 @@ def get_medec_run_spec() -> RunSpec:
 
 
 @run_spec_function("ehrshot")
-def get_ehrshot_spec(subject: str, max_length: int = 100000) -> RunSpec:
+def get_ehrshot_spec(subject: str, data_path: str, max_length: int = 100000) -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.ehrshot_scenario.EHRSHOTScenario",
-        args={"subject": subject, "max_length": max_length},
+        args={
+            "subject": subject,
+            "max_length": max_length,
+            "data_path": data_path,
+        },
     )
 
     adapter_spec = get_multiple_choice_adapter_spec(
@@ -320,9 +330,13 @@ def get_medbullets_freetext_run_spec() -> RunSpec:
 
 
 @run_spec_function("medalign")
-def get_medalign_spec(max_length: int = 40000) -> RunSpec:
+def get_medalign_spec(data_path: str, max_length: int = 40000) -> RunSpec:
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.medalign_scenario.MedalignScenario", args={"max_length": max_length}
+        class_name="helm.benchmark.scenarios.medalign_scenario.MedalignScenario",
+        args={
+            "max_length": max_length,
+            "data_path": data_path,
+        },
     )
 
     adapter_spec = get_generation_adapter_spec(
@@ -358,8 +372,11 @@ def get_medalign_spec(max_length: int = 40000) -> RunSpec:
 
 
 @run_spec_function("shc_ptbm_med")
-def get_shc_ptbm_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.shc_ptbm_scenario.SHCPTBMMedScenario", args={})
+def get_shc_ptbm_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.shc_ptbm_scenario.SHCPTBMMedScenario",
+        args={"data_path": data_path},
+    )
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
@@ -378,8 +395,11 @@ def get_shc_ptbm_spec() -> RunSpec:
 
 
 @run_spec_function("shc_sei_med")
-def get_shc_sei_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.shc_sei_scenario.SHCSEIMedScenario", args={})
+def get_shc_sei_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.shc_sei_scenario.SHCSEIMedScenario",
+        args={"data_path": data_path},
+    )
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
@@ -398,8 +418,13 @@ def get_shc_sei_spec() -> RunSpec:
 
 
 @run_spec_function("dischargeme")
-def get_dischargeme_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.dischargeme_scenario.DischargeMeScenario")
+def get_dischargeme_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.dischargeme_scenario.DischargeMeScenario",
+        args={
+            "data_path": data_path,
+        },
+    )
 
     adapter_spec = get_generation_adapter_spec(
         instructions=(
@@ -534,8 +559,11 @@ def get_mtsamples_procedures_spec() -> RunSpec:
 
 
 @run_spec_function("mimic_rrs")
-def get_mimic_rrs_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.mimic_rrs_scenario.MIMICRRSScenario", args={})
+def get_mimic_rrs_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.mimic_rrs_scenario.MIMICRRSScenario",
+        args={"data_path": data_path},
+    )
 
     adapter_spec = get_generation_adapter_spec(
         instructions=(
@@ -572,8 +600,11 @@ def get_mimic_rrs_spec() -> RunSpec:
 
 
 @run_spec_function("mimic_bhc")
-def get_mimic_bhc_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.mimic_bhc_scenario.MIMICBHCScenario", args={})
+def get_mimic_bhc_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.mimic_bhc_scenario.MIMICBHCScenario",
+        args={"data_path": data_path},
+    )
 
     adapter_spec = get_generation_adapter_spec(
         instructions=("Summarize the clinical note into a brief hospital course."),
@@ -585,23 +616,29 @@ def get_mimic_bhc_spec() -> RunSpec:
         max_train_instances=0,
         stop_sequences=[],
     )
+    annotator_specs = [AnnotatorSpec(class_name="helm.benchmark.annotation.mimic_bhc_annotator.MIMICBHCAnnotator")]
+
     metric_args = {
         "task": "mimic_bhc",
         "device": get_torch_device_name(),
         "bertscore_model": "distilbert-base-uncased",
         "rescale_with_baseline": False,
     }
+    metric_specs = get_summarization_metric_specs(metric_args) + [
+        MetricSpec(class_name="helm.benchmark.metrics.mimic_bhc_metrics.MIMICBHCMetric", args={})
+    ]
     return RunSpec(
         name="mimic_bhc",
+        annotators=annotator_specs,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
-        metric_specs=get_summarization_metric_specs(metric_args),
+        metric_specs=metric_specs,
         groups=["mimic_bhc"],
     )
 
 
 @run_spec_function("chw_care_plan")
-def get_chw_care_plan_run_spec() -> RunSpec:
+def get_chw_care_plan_run_spec(data_path: str) -> RunSpec:
     """
     RunSpec for the chw_care_plan dataset.
     This configuration evaluates the model's ability to summarize
@@ -609,7 +646,7 @@ def get_chw_care_plan_run_spec() -> RunSpec:
     """
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.chw_care_plan_scenario.CHWCarePlanScenario",
-        args={},
+        args={"data_path": data_path},
     )
 
     adapter_spec = get_generation_adapter_spec(
@@ -681,10 +718,10 @@ def get_medication_qa_spec() -> RunSpec:
 
 
 @run_spec_function("starr_patient_instructions")
-def get_starr_patient_instructions_run_spec() -> RunSpec:
+def get_starr_patient_instructions_run_spec(data_path: str) -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.starr_patient_instructions_scenario.StarrPatientInstructionsScenario",
-        args={},
+        args={"data_path": data_path},
     )
 
     adapter_spec = get_generation_adapter_spec(
@@ -748,6 +785,7 @@ def get_med_dialog_spec(subset: str) -> RunSpec:
         output_noun="Summary",
         max_tokens=80,
         max_train_instances=0,
+        stop_sequences=[],
     )
     annotator_specs = [AnnotatorSpec(class_name="helm.benchmark.annotation.med_dialog_annotator.MedDialogAnnotator")]
 
@@ -771,8 +809,11 @@ def get_med_dialog_spec(subset: str) -> RunSpec:
 
 
 @run_spec_function("shc_conf_med")
-def get_shc_conf_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.shc_conf_scenario.SHCCONFMedScenario", args={})
+def get_shc_conf_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.shc_conf_scenario.SHCCONFMedScenario",
+        args={"data_path": data_path},
+    )
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
@@ -824,13 +865,16 @@ def get_medi_qa_spec() -> RunSpec:
 
 
 @run_spec_function("mental_health")
-def get_mental_health_spec() -> RunSpec:
+def get_mental_health_spec(data_path: str) -> RunSpec:
     """
     Returns the run specification for the mental health counseling scenario.
     This scenario evaluates a model's ability to generate appropriate counseling responses
     in mental health conversations.
     """
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.mental_health_scenario.MentalHealthScenario")
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.mental_health_scenario.MentalHealthScenario",
+        args={"data_path": data_path},
+    )
 
     adapter_spec = get_generation_adapter_spec(
         instructions=(
@@ -840,6 +884,7 @@ def get_mental_health_spec() -> RunSpec:
         newline_after_input_noun=False,
         output_noun="Counselor response",
         max_tokens=512,
+        stop_sequences=[],
     )
     annotator_specs = [
         AnnotatorSpec(class_name="helm.benchmark.annotation.mental_health_annotator.MentalHealthAnnotator")
@@ -871,7 +916,11 @@ def get_pubmed_qa_spec() -> RunSpec:
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
-        instructions="Answer A for yes, B for no or C for maybe.",
+        instructions=(
+            "Answer A for yes, B for no or C for maybe. "
+            "Do not include any explanation or additional text. "
+            "Output only the letter on a single line."
+        ),
         input_noun="Question",
         output_noun="Answer",
         max_train_instances=0,
@@ -937,8 +986,11 @@ def get_ehr_sql_run_spec() -> RunSpec:
 
 
 @run_spec_function("shc_bmt_med")
-def get_shc_bmt_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.shc_bmt_scenario.SHCBMTMedScenario", args={})
+def get_shc_bmt_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.shc_bmt_scenario.SHCBMTMedScenario",
+        args={"data_path": data_path},
+    )
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
@@ -1002,6 +1054,7 @@ No letter or word, just the integer value.
 Your Judgment"""  # noqa: E501
         ),
         max_train_instances=0,
+        stop_sequences=[],
     )
 
     return RunSpec(
@@ -1014,17 +1067,17 @@ Your Judgment"""  # noqa: E501
 
 
 @run_spec_function("n2c2_ct_matching")
-def get_n2c2_ct_matching_spec(subject: str) -> RunSpec:
+def get_n2c2_ct_matching_spec(data_path: str, subject: str) -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.n2c2_ct_matching_scenario.N2C2CTMatchingScenario",
-        args={"subject": subject},
+        args={"data_path": data_path, "subject": subject},
     )
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
         instructions="Answer A for yes, B for no.",
         input_noun="",
-        output_noun="Answer A for yes, B for no",
+        output_noun="Answer A for yes, B for no. Do not add any other text, punctuation, or symbols",
         max_train_instances=0,
     )
 
@@ -1038,8 +1091,10 @@ def get_n2c2_ct_matching_spec(subject: str) -> RunSpec:
 
 
 @run_spec_function("shc_gip_med")
-def get_shc_gip_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.shc_gip_scenario.SHCGIPMedScenario", args={})
+def get_shc_gip_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.shc_gip_scenario.SHCGIPMedScenario", args={"data_path": data_path}
+    )
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
@@ -1058,11 +1113,11 @@ def get_shc_gip_spec() -> RunSpec:
 
 
 @run_spec_function("mimiciv_billing_code")
-def get_mimiciv_billing_code_spec() -> RunSpec:
+def get_mimiciv_billing_code_spec(data_path: str) -> RunSpec:
     scenario_spec = ScenarioSpec(
         class_name="helm.benchmark.scenarios.mimiciv_billing_code_scenario.MIMICIVBillingCodeScenario",
         args={
-            "data_file": "/share/pi/nigam/data/medhelm/mimiciv_billing_codes/mimiciv_icd10.feather",
+            "data_path": data_path,
         },
     )
     adapter_spec = get_generation_adapter_spec(
@@ -1094,9 +1149,9 @@ def get_mimiciv_billing_code_spec() -> RunSpec:
 
 
 @run_spec_function("shc_sequoia_med")
-def get_shc_sequoia_spec() -> RunSpec:
+def get_shc_sequoia_spec(data_path: str) -> RunSpec:
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.shc_sequoia_scenario.SHCSequoiaMedScenario", args={}
+        class_name="helm.benchmark.scenarios.shc_sequoia_scenario.SHCSequoiaMedScenario", args={"data_path": data_path}
     )
 
     adapter_spec = get_multiple_choice_adapter_spec(
@@ -1116,8 +1171,10 @@ def get_shc_sequoia_spec() -> RunSpec:
 
 
 @run_spec_function("shc_cdi_med")
-def get_shc_cdi_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.shc_cdi_scenario.SHCCDIMedScenario", args={})
+def get_shc_cdi_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.shc_cdi_scenario.SHCCDIMedScenario", args={"data_path": data_path}
+    )
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
@@ -1136,8 +1193,10 @@ def get_shc_cdi_spec() -> RunSpec:
 
 
 @run_spec_function("shc_ent_med")
-def get_shc_ent_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.shc_ent_scenario.SHCENTMedScenario", args={})
+def get_shc_ent_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.shc_ent_scenario.SHCENTMedScenario", args={"data_path": data_path}
+    )
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
@@ -1153,3 +1212,49 @@ def get_shc_ent_spec() -> RunSpec:
         metric_specs=get_exact_match_metric_specs(),
         groups=["shc_ent_med"],
     )
+
+
+@run_spec_function("shc_privacy_med")
+def get_shc_privacy_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.shc_cdi_scenario.SHCPRIVACYMedScenario",
+        args={"data_path": data_path},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="Answer A or B.",
+        input_noun="",
+        output_noun="",
+    )
+
+    return RunSpec(
+        name="shc_privacy_med",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["shc_privacy_med"],
+    )
+
+
+@run_spec_function("shc_proxy_med")
+def get_shc_proxy_spec(data_path: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.shc_cdi_scenario.SHCPROXYMedScenario",
+        args={"data_path": data_path},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="Answer A or B.",
+        input_noun="",
+        output_noun="",
+    )
+
+    return RunSpec(
+        name="shc_proxy_med",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["shc_proxy_med"],
+    )