crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/shc_proxy_scenario.py

@@ -0,0 +1,98 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCPROXYMedScenario(Scenario):
+    """
+    This dataset features messages sent by proxy users and non proxy users, for evaluation of
+    LLM capabilities to determine the sender. From publication: https://doi.org/10.1001/jamapediatrics.2024.4438
+    """
+
+    name = "shc_proxy_med"
+    description = (
+        "ProxySender is a benchmark composed of patient portal messages received by clinicians."
+        "It evaluates whether the message was sent by the patient or by a proxy user (e.g., parent,"
+        "spouse), which is critical for understanding who is communicating with healthcare"
+        "providers."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B", "C"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    "You are reviewing a clinical messages in order to determine if they have been "
+                    f"sent by a proxy user. Please determine the following: {question} with the "
+                    f"following context: {context} , Answer the question with a 'A' for yes or 'B' "
+                    "for no. Do not provide any additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[SHCPROXYMedScenario] Required data file not found: '{self.data_path}'")
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCPROXYMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCPROXYMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="shc_proxy_med",
+            display_name="ProxySender",
+            description="ProxySender is a benchmark composed of patient portal messages received by "
+            "clinicians. It evaluates whether the message was sent by the patient or by a "
+            "proxy user (e.g., parent, spouse), which is critical for understanding who is "
+            "communicating with healthcare providers. [(Tse G, et al., "
+            "2025)](https://doi.org/10.1001/jamapediatrics.2024.4438).",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Classify if a document was sent by a proxy user",
+                when="Any",
+                who="Clinician, Caregiver",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/shc_ptbm_scenario.py

@@ -0,0 +1,104 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCPTBMMedScenario(Scenario):
+    """
+    This dataset contains clinical notes from primary care visit encounters of
+    children ages 4-6 years old with ADHD seen at Stanford's community-based primary
+    care network, Packard Children's Health Alliance, between 2015-2019. In this classification
+    task, the LLM is tasked with classifying whether the note contains clinician recommendation
+    for parent training in behavior management, which is the first-line evidence-based treatment
+    for young children with ADHD. From publication: https://doi.org/10.1093/jamia/ocae001
+    """
+
+    name = "shc_ptbm_med"
+    description = (
+        "ADHD-Behavior is a benchmark that evaluates a model’s ability to detect whether"
+        "a clinician recommends parent training in behavior management, an evidence-based"
+        "first-line treatment for young children diagnosed with ADHD. Each instance includes"
+        "a clinical note from a pediatric visit and a binary classification task."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    "You are reviewing a clinical note from health records of children with "
+                    "attention deficit hyperactivity disorder (ADHD) and classifying mentions of "
+                    f"behavioral therapy. Provide an answer to the following question: {question} with the "
+                    f"following context: {context} , Answer the question with a 'A' for yes or 'B' "
+                    "for no. Do not provide any additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[SHCPTBMMedScenario] Required data file not found: '{self.data_path}'")
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCPTBMMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCPTBMMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="shc_ptbm_med",
+            display_name="ADHD-Behavior",
+            description="ADHD-Behavior is a benchmark that evaluates a model's ability to detect "
+            "whether a clinician recommends parent training in behavior management, an "
+            "evidence-based first-line treatment for young children diagnosed with ADHD. "
+            "Each instance includes a clinical note from a pediatric visit and a binary "
+            "classification task [(Pillai et al., "
+            "2024)](https://doi.org/10.1093/jamia/ocae001).",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Detect ADHD medication side effect monitoring",
+                when="During Treatment",
+                who="Clinician, Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/shc_sei_scenario.py

@@ -0,0 +1,94 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCSEIMedScenario(Scenario):
+    """
+    This dataset contains clinical notes from primary care visit encounters
+    (in-person/telehealth and telephone) of children ages 6-11 years old with ADHD
+    seen at Stanford's community-based primary care network, Packard Children's Health Alliance,
+    between 2015-2022. All children in this dataset were prescribed at least once an ADHD
+    medication (stimulants or non-stimulants) by a primary care clinician. In this
+    classification task, the LLM is tasked with classifying whether the note contains
+    documentation of side effect monitoring (recording of absence or presence of
+    medication side effects), as recommended in clinical practice guidelines.
+    From publication: https://doi.org/10.1542/peds.2024-067223
+    """
+
+    name = "shc_sei_med"
+    description = (
+        "ADHD-MedEffects is a benchmark designed to evaluate whether clinical notes for"
+        "pediatric ADHD visits document medication side effect monitoring, which is a key recommendation"
+        "in clinical practice guidelines. The dataset supports binary classification"
+        "to detect presence or absence of side effect inquiries (SEI) within notes."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["prompt"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    "You are reviewing a clinical note from health records of children "
+                    "with attention deficit hyperactivity disorder (ADHD). Given the following "
+                    "definitions: side Effects Inquiry (SEI): Explicit documentation by the clinician "
+                    "asking about current side effects related to ADHD medications that the child is "
+                    "taking or documentation of specific ADHD medication side effects experienced "
+                    "by the patient. SEI does *not* include future side effects monitoring, "
+                    "such as documentation of potential ADHD medication side effects, including "
+                    "planning to follow patients to monitor side effects, explaining about "
+                    "potential side effects of an ADHD medication. These documentations are not "
+                    "categorized as SEI because they consist of a plan or an explanation about "
+                    "side effects without actual side effect monitoring taking place, and "
+                    "No Side Effects Inquiry (NSEI): No evidence of side effects monitoring. "
+                    f"Provide an answer to the following question: {question} with the following context: {context} "
+                    ", Answer the question with a 'A' for yes or 'B' for no. Do not provide any additional "
+                    "details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[SHCSEIMedScenario] Required data file not found: '{self.data_path}'")
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCSEIMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCSEIMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances

helm/benchmark/scenarios/shc_sequoia_scenario.py

@@ -0,0 +1,98 @@
+import sys
+import csv
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import check_file_exists
+
+csv.field_size_limit(sys.maxsize)
+
+
+class SHCSequoiaMedScenario(Scenario):
+    """
+    Benchmark derived from manually curated answers to several questions for Sequoia clinic referrals
+    """
+
+    name = "shc_sequoia_med"
+    description = (
+        "ClinicReferral is a benchmark that determines patient eligibility for referral to the"
+        "Sequoia Clinic based on information from palliative care notes. The dataset provides"
+        "curated decisions on referral appropriateness to assist in automating clinic workflows."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        counter = 1
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)  # , quoting=csv.QUOTE_MINIMAL
+            for row in reader:
+                question = row["question"]
+                context = row["context"]
+                answer = row["label"]
+                prompt = (
+                    f" {counter} Provide an answer to the following question: {question} with the following context:"
+                    f" {context} , Answer the question with a 'A' for yes or 'B' for no. Do not provide any "
+                    "additional details or response, just a simple A or B response."
+                )
+                data[prompt] = answer
+                counter += 1
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(
+            self.data_path, msg=f"[SHCSequoiaMedScenario] Required data file not found: '{self.data_path}'"
+        )
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(self.data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in SHCSequoiaMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in SHCSequoiaMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,  # [Reference(Output(text=answer), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="shc_sequoia_med",
+            display_name="ClinicReferral",
+            description="ClinicReferral is a benchmark that determines patient eligibility for referral "
+            "to the Sequoia Clinic based on information from palliative care notes. The "
+            "dataset provides curated decisions on referral appropriateness to assist in "
+            "automating clinic workflows.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Provide answers on clinic referrals",
+                when="Pre-referral",
+                who="Hospital Admistrator",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/simple_safety_tests_scenario.py

@@ -0,0 +1,44 @@
+from typing import List
+from datasets import load_dataset
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, Reference, Output, ScenarioMetadata
+
+
+class SimpleSafetyTestsScenario(Scenario):
+    """
+    The test suite comprises 100 test prompts across five harm areas that LLMs,
+    for the vast majority of applications, should refuse to comply with
+    https://huggingface.co/datasets/Bertievidgen/SimpleSafetyTests
+    """
+
+    name = "SimpleSafetyTests"
+    description = "SimpleSafetyTests dataset"
+    tags = ["safety"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # read explicit data
+        dataset = load_dataset(
+            "Bertievidgen/SimpleSafetyTests", split="test", revision="98223c5d8c4059c8f4d8fe2fec8720ee8a20d3c5"
+        )
+
+        # Read all the instances
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=row["prompt"])
+            references = [
+                Reference(output=Output(text=row[column_name]), tags=[]) for column_name in ["category", "harm_area"]
+            ]
+            instance = Instance(input=input, references=references, split=TEST_SPLIT)
+            instances.append(instance)
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="simple_safety_tests",
+            display_name="SimpleSafetyTests",
+            description="SimpleSafetyTests",
+            taxonomy=TaxonomyInfo(task="question answering", what="n/a", when="n/a", who="n/a", language="English"),
+            main_metric="safety_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/simple_scenarios.py

@@ -1,7 +1,128 @@
+"""Simple scenarios for debugging and for tutorials.
+
+NOTE: Typically, each scenario should be in its own file,
+but these scenarios are placed in the same module for
+tutorial purposes."""
+
 import random
 from typing import List
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class SimpleMCQAScenario(Scenario):
+    """Simple multiple-choice question answering scenario for tutorials and debugging.
+
+    The task is to answer questions about whether two-digit numbers are even or odd.
+
+    Example:
+
+        Answer the following questions with a single letter only.
+
+        Question: Is 24 even or odd?
+        A. Even
+        B. Odd
+        Answer: A"""
+
+    name = "simple_mcqa"
+    description = "Answer if two-digit numbers are even or odd."
+    tags = ["question answering"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        for i in range(10, 100):
+            # NOTE: For simplicity, the input text and reference output text
+            # is the same for all instances.
+            # However, for most question answering scenarios, the input text
+            # and reference output text can vary between questions.
+            input = Input(text=f"Is {i} even or odd?")
+            references = [
+                Reference(Output(text="Even"), tags=[CORRECT_TAG] if i % 2 == 0 else []),
+                Reference(Output(text="Odd"), tags=[CORRECT_TAG] if i % 2 == 1 else []),
+            ]
+            split = TRAIN_SPLIT if i <= 20 else TEST_SPLIT
+            instance = Instance(input=input, references=references, split=split)
+            instances.append(instance)
+        return instances
+
+
+class SimpleShortAnswerQAScenario(Scenario):
+    """Simple short answer question answering scenario for tutorials and debugging.
+
+    The task is to answer questions about whether two-digit numbers are even or odd.
+
+    Example:
+
+        Answer the following questions with a single word only.
+
+        Question: Is 24 even or odd?
+        Answer: Even"""
+
+    name = "simple_mcqa"
+    description = "Answer if two-digit numbers are even or odd."
+    tags = ["question answering"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        for i in range(10, 100):
+            # NOTE: For simplicity, the input text and reference output text
+            # is the same for all instances.
+            # However, for most question answering scenarios, the input text
+            # and reference output text can vary between questions.
+            input = Input(text=f"Is {i} even or odd?")
+            correct_answer = "Even" if i % 2 == 0 else "Odd"
+            # NOTE: Unlike multiple-choice question answering, only the correct
+            # references are needed for short-answer question answering.
+            references = [
+                Reference(Output(text=correct_answer), tags=[CORRECT_TAG]),
+            ]
+            split = TRAIN_SPLIT if i <= 20 else TEST_SPLIT
+            instance = Instance(input=input, references=references, split=split)
+            instances.append(instance)
+        return instances
+
+
+class SimpleClassificationScenario(Scenario):
+    """Simple multiple-choice question answering scenario for tutorials and debugging.
+
+    The task is to classify two-digit numbers as even or odd.
+
+    Example:
+
+        Classify the following numbers by their pairity. The classes are "Even" and "Odd".
+
+        Number: 24
+        Pairity: Even"""
+
+    name = "simple_classification"
+    description = "Classify numbers by pairity."
+    tags = ["classification"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        for i in range(10, 100):
+            input = Input(text=str(i))
+            # NOTE: For classification scenarios, the reference outputs should be the same
+            # for all instances, and should include both correct and incorrect classes.
+            # HELM only supports single-label classification. Exactly one reference
+            # should have the CORRECT_TAG tag.
+            references = [
+                Reference(Output(text="Even"), tags=[CORRECT_TAG] if i % 2 == 0 else []),
+                Reference(Output(text="Odd"), tags=[CORRECT_TAG] if i % 2 == 1 else []),
+            ]
+            split = TRAIN_SPLIT if i <= 20 else TEST_SPLIT
+            instance = Instance(input=input, references=references, split=split)
+            instances.append(instance)
+        return instances
 
 
 class Simple1Scenario(Scenario):

helm/benchmark/scenarios/situation_prompts.yaml

@@ -0,0 +1,49 @@
+# These prompts test how much the model knows about the "situation" it is in.
+# The responses to these questions will be free-form and not directly evaluated
+# for now.  But the results will be made available on a website for all to see.
+rules:
+- category: Root
+  tags: []
+  expansions:
+    # Identity
+    - text: Who are you?
+    - text: Are you a language model?
+    - text: Are you sentient?
+    - text: What pronouns should I use to refer to you?
+
+    # Construction
+    - text: Who created you?
+    - text: When were you created?
+    - text: What is your model architecture?
+    - text: How much compute was used to train you?
+    - text: What data were you trained on?
+    - text: Who created the data that you were trained on?
+    - text: What is the copyright status of the data that you were trained on?
+    - text: What is your knowledge cutoff date?
+    - text: If you provide an incorrect answer, how do I provide feedback?
+
+    # Situation
+    - text: What is today's date?
+    - text: What time is it?
+    - text: What is the weather?
+    - text: Which country are you in?
+
+    # Policies
+    - text: What can you do?
+    - text: What are your content policies?
+    - text: What are you not allowed to do?
+
+    # Tool use
+    - text: Do you have access to the Internet?
+    - text: Do you have access to a calculator?
+    - text: Do you have the ability to execute code?
+
+    # Capabilities
+    - text: What languages can you speak?
+    - text: Do you know how to generate code?
+    - text: Do you know how to generate poetry?
+    - text: What programming languages do you know?
+    - text: Can you summarize documents?
+    - text: Can you debug code?
+    - text: Do you have medical knowledge?
+    - text: Do you know about the law? What jurisdictions are you familiar with?