crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/self_instruct_scenario.py

@@ -2,8 +2,18 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import CORRECT_TAG, Reference, Scenario, Instance, Input, TEST_SPLIT, Output
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Reference,
+    Scenario,
+    Instance,
+    Input,
+    TEST_SPLIT,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class SelfInstructScenario(Scenario):
@@ -46,3 +56,21 @@ class SelfInstructScenario(Scenario):
                 )
                 instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="self_instruct",
+            display_name="Self Instruct",
+            short_display_name="Self Instruct",
+            description="The manually-curated instructions from the Self-Instruct paper ([Wang et al., "
+            "2023](https://aclanthology.org/2023.acl-long.754.pdf)).",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Instructions for LLMs",
+                when="2022",
+                who="Authors of the research paper",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/shc_bmt_scenario.py

@@ -0,0 +1,97 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCBMTMedScenario(Scenario):
+    """
+    This benchmark dataset was built from a patient status gold-standard
+    for specific questions asked after a bone marrow transplant has taken place.
+    """
+
+    name = "shc_bmt_med"
+    description = (
+        "BMT-Status is a benchmark composed of clinical notes and associated binary questions"
+        "related to bone marrow transplant (BMT), hematopoietic stem cell transplant (HSCT),"
+        "or hematopoietic cell transplant (HCT) status. The goal is to determine whether the"
+        "patient received a subsequent transplant based on the provided clinical documentation."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f"Provide an answer to the following question: {question} with the following context: {context} "
+                    ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
+                    "details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[SHCBMTMedScenario] Required data file not found: '{self.data_path}'")
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCBMTMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCBMTMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="shc_bmt_med",
+            display_name="BMT-Status",
+            description="BMT-Status is a benchmark composed of clinical notes and associated binary "
+            "questions related to bone marrow transplant (BMT), hematopoietic stem cell "
+            "transplant (HSCT), or hematopoietic cell transplant (HCT) status. The goal is "
+            "to determine whether the patient received a subsequent transplant based on the "
+            "provided clinical documentation.",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="Answer bone marrow transplant questions",
+                when="Any",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/shc_cdi_scenario.py

@@ -0,0 +1,95 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCCDIMedScenario(Scenario):
+    """
+    This benchmark dataset was built from Clinical Document Integrity (CDI)
+    notes were there are verifications of clinical activities. The idea behind
+    it was to assess an LLM capability to answer these questions from previous notes.
+    """
+
+    name = "shc_cdi_med"
+    description = (
+        "CDI-QA is a benchmark constructed from Clinical Documentation Integrity (CDI)"
+        "notes. It is used to evaluate a model's ability to verify clinical conditions based on"
+        "documented evidence in patient records."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f"Provide an answer to the following question: {question} with the following context: {context} , "
+                    "Answer the question with either 'A' for yes or 'B' for no. Do not provide any "
+                    "additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[SHCCDIMedScenario] Required data file not found: '{self.data_path}'")
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCCDIMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCCDIMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="shc_cdi_med",
+            display_name="CDI-QA",
+            description="CDI-QA is a benchmark constructed from Clinical Documentation Integrity (CDI) "
+            "notes. It is used to evaluate a model's ability to verify clinical conditions "
+            "based on documented evidence in patient records.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Answer verification questions from CDI notes",
+                when="Any",
+                who="Hospital Admistrator",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/shc_conf_scenario.py

@@ -0,0 +1,99 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCCONFMedScenario(Scenario):
+    """
+    Benchmark derived from extracting confidential information from clinical notes.
+    From Evaluation of a Large Language Model to Identify Confidential Content in
+    Adolescent Encounter Notes published at https://jamanetwork.com/journals/jamapediatrics/fullarticle/2814109
+    """
+
+    name = "shc_conf_med"
+    description = (
+        "MedConfInfo is a benchmark comprising clinical notes from adolescent patients. It is"
+        "used to evaluate whether the content contains sensitive protected health information"
+        "(PHI) that should be restricted from parental access, in accordance with adolescent"
+        "confidentiality policies in clinical care."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f"Provide an answer to the following question: {question} with the following context: {context} "
+                    ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
+                    "details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[SHCCONFMedScenario] Required data file not found: '{self.data_path}'")
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCCONFMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCCONFMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="shc_conf_med",
+            display_name="MedConfInfo",
+            description="MedConfInfo is a benchmark comprising clinical notes from adolescent patients. "
+            "It is used to evaluate whether the content contains sensitive protected health "
+            "information (PHI) that should be restricted from parental access, in "
+            "accordance with adolescent confidentiality policies in clinical care. "
+            "[(Rabbani et al., "
+            "2024)](https://jamanetwork.com/journals/jamapediatrics/fullarticle/2814109).",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Identify sensitive health info in adolescent notes",
+                when="Any",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/shc_ent_scenario.py

@@ -0,0 +1,98 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCENTMedScenario(Scenario):
+    """
+    This benchmark dataset was built to assess the capabilities "
+    "of an LLM for referral to the Ear, Nose and Throat department.
+    """
+
+    name = "shc_ent_med"
+    description = (
+        "ENT-Referral is a benchmark designed to evaluate whether a patient's clinical note"
+        "supports a referral to an Ear, Nose, and Throat (ENT) specialist. It helps assess"
+        "models' abilities to make referral decisions based on unstructured clinical text."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B", "C"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        counter = 1
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                if row["label"] != "":  # skip rows with character/encoding issues - 79
+                    question = row["prompt"]
+                    context = row["context"]
+                    answer = row["label"]
+                    prompt = (
+                        f"{counter} Provide an answer to the following question: {question} with the following context:"
+                        f" {context} , Answer the question with either 'A' for yes, 'B' for no, or 'C' for no mention."
+                        " Do not provide any additional details or response, just a simple A, B, or C response."
+                    )
+                    data[prompt] = answer
+                    counter = counter + 1
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[SHCENTMedScenario] Required data file not found: '{self.data_path}'")
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCENTMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCENTMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="shc_ent_med",
+            display_name="ENT-Referral",
+            description="ENT-Referral is a benchmark designed to evaluate whether a patient's clinical "
+            "note supports a referral to an Ear, Nose, and Throat (ENT) specialist. It "
+            "helps assess models' abilities to make referral decisions based on "
+            "unstructured clinical text",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Identify referrals for ENT specialists",
+                when="Any",
+                who="Hospital Admistrator",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/shc_gip_scenario.py

@@ -0,0 +1,94 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCGIPMedScenario(Scenario):
+    """
+    This benchmark dataset was built from a patient referral gold-standard set
+    to a specialty clinic to verify the ability of LLMs for patient hospice referral purposes.
+    """
+
+    name = "shc_gip_med"
+    description = (
+        "HospiceReferral is a benchmark that evaluates model performance in identifying"
+        "whether patients are eligible for hospice care based on palliative care clinical notes."
+        "The benchmark focuses on end-of-life care referral decisions."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f"Provide an answer to the following question: {question} with the following context: {context} "
+                    ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
+                    "details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[SHCGIPMedScenario] Required data file not found: '{self.data_path}'")
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCGIPMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCGIPMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="shc_gip_med",
+            display_name="HospiceReferral",
+            description="HospiceReferral is a benchmark that evaluates model performance in identifying "
+            "whether patients are eligible for hospice care based on palliative care "
+            "clinical notes. The benchmark focuses on end-of-life care referral decisions.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Assess hospice referral appropriateness",
+                when="End-of-care",
+                who="Hospital Admistrator",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/shc_privacy_scenario.py

@@ -0,0 +1,100 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCPRIVACYMedScenario(Scenario):
+    """
+    This dataset features messages sent generated by an LLM from patient clinical notes data.
+    The scenario evaluates the ability of an LLM to determine if any potentially confidential
+    information about the patient was included. From publication: https://doi.org/10.1001/jamapediatrics.2024.4438
+    """
+
+    name = "shc_privacy_med"
+    description = (
+        "PrivacyDetection is a benchmark composed of patient portal messages submitted by"
+        "patients or caregivers. The task is to determine whether the message contains any"
+        "confidential or privacy-leaking information that should be protected."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    "You are reviewing clinical messages in order to determine if any confidential "
+                    f"information was included. Please answer the following question: {question} with the "
+                    f"following context: {context} , Answer the question with a 'A' for yes or 'B' "
+                    "for no. Do not provide any additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(
+            self.data_path, msg=f"[SHCPRIVACYMedScenario] Required data file not found: '{self.data_path}'"
+        )
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCPRIVACYMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCPRIVACYMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="shc_privacy_med",
+            display_name="PrivacyDetection",
+            description="PrivacyDetection is a benchmark composed of patient portal messages submitted "
+            "by patients or caregivers. The task is to determine whether the message "
+            "contains any confidential or privacy-leaking information that should be "
+            "protected [(Tse G, et al., "
+            "2025)](https://doi.org/10.1001/jamapediatrics.2024.4438).",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Classify if a document leaks private information",
+                when="Any",
+                who="Clinician, Caregiver",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )