crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/scenarios/mental_health_scenario.py

@@ -0,0 +1,112 @@
+import pandas as pd
+from typing import List
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    PassageQuestionInput,
+    Output,
+)
+
+
+class MentalHealthScenario(Scenario):
+    """
+    This scenario evaluates language models' ability to generate appropriate counseling responses
+    in mental health conversations. The dataset contains counseling dialogues covering
+    various topics including workplace issues, anxiety, suicidal thoughts, relationship
+    problems, and more.
+
+    Each dialogue consists of interactions between a counselor and a client, where the counselor
+    demonstrates expert mental health counseling techniques. The dialogues were selected based on high
+    quality scores from multiple evaluators.
+
+    Example dialogue structure:
+    ```
+    counselor: Hi there, to start can you tell me your name and a little bit about what's been going on?
+    client: I sleep too much... I'm 23, female and work as IT professional. I feel like I'm not fitting in...
+    counselor: I can see you have been facing challenges with feeling like you don't fit in...
+    ```
+
+    The task is to generate the next counselor response given the conversation history. Models
+    are evaluated on their ability to:
+    1. Provide empathetic and supportive responses
+    2. Follow proper mental health counseling protocols
+    3. Generate contextually appropriate interventions
+
+    The dataset includes:
+    - 7 complete dialogues covering different mental health topics
+    - Metadata about dialogue topic and type
+    - Gold-standard counselor responses as references
+    - Full conversation history for context
+
+    Each instance includes:
+    - input: Previous conversation turns formatted with speaker labels
+    - reference: The actual counselor's response (gold standard)
+    - metadata: Topic and type of mental health conversation
+    """
+
+    name = "mental_health"
+    description = "A dataset containing a counselor and mental health patient conversation, where the objective is to \
+                    generate an empathetic counselor response."
+    tags = ["dialogue", "counseling", "mental_health", "empathy", "healthcare"]
+
+    def process_dialogue_data(self, data: pd.DataFrame) -> List[Instance]:
+        """
+        Process the dialogue data into evaluation instances.
+        Each instance represents a point in the conversation where the model needs to generate
+        a counselor response.
+
+        Args:
+            data (pd.DataFrame): DataFrame containing processed dialogues with columns:
+                - context: Previous conversation history
+                - gold_counselor_response: Reference counselor response
+                - topic: Type of conversation (e.g., "Anxiety", "Workplace")
+                - dialogue_type: Numerical type identifier
+
+        Returns:
+            List[Instance]: List of processed instances ready for evaluation
+        """
+        instances: List[Instance] = []
+
+        for _, row in data.iterrows():
+            # Format input with clear section breaks and instructions
+            input_text = (
+                f"Topic: {row['topic']}\n"
+                f"Type: {row['dialogue_type']}\n"
+                f"\nPrevious conversation:\n{row['context']}\n"
+                f"\nGenerate an empathetic and appropriate counselor response:"
+            )
+
+            # Create input with empty passage since all context is in question
+            prompt = PassageQuestionInput(passage="", question=input_text)
+
+            # Create instance with gold standard response
+            instance = Instance(
+                input=prompt,
+                references=[Reference(Output(text=row["gold_counselor_response"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Load and process the mental health dialogue dataset.
+
+        Args:
+            output_path (str): Path for any cached or intermediate files
+
+        Returns:
+            List[Instance]: List of processed instances for evaluation
+        """
+        # Load the processed dialogue data
+        data_path = "/share/pi/nigam/data/medhelm/mental_health/processed_dialogues.csv"
+        dialogue_data = pd.read_csv(data_path)
+
+        # Process into instances
+        instances = self.process_dialogue_data(dialogue_data)
+
+        return instances

helm/benchmark/scenarios/mimic_bhc_scenario.py

@@ -0,0 +1,98 @@
+import json
+from typing import Dict, List
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+
+class MIMICBHCScenario(Scenario):
+    """
+    MIMIC-IV-BHC presents a curated collection of preprocessed discharge notes with labeled brief hospital
+    course (BHC) summaries. This dataset is derived from MIMIC-IV (https://doi.org/10.1093/jamia/ocae312).
+
+    In total, the dataset contains 270,033 clinical notes.
+    The splits are provided by the dataset itself.
+
+    Sample Synthetic Prompt:
+        Summarize the clinical note into a brief hospital course.
+
+        Clinical Note:
+        <SEX> M <SERVICE> SURGERY <ALLERGIES> No Known Allergies \/ Adverse Drug Reactions
+        ...
+        continue to follow-up with your health care providers as an outpatient.
+
+        Brief Hospital Course:
+        Mr. ___ was pre-admitted on ___ for liver transplantation
+        ...
+        discharged home to continue home medications and follow-up as an outpatient.
+
+    @article{aali2024dataset,
+        title={A dataset and benchmark for hospital course summarization with adapted large language models},
+        author={Aali, Asad and Van Veen, Dave and Arefeen, YI and Hom, Jason and Bluethgen, Christian
+        and Reis, Eduardo Pontes and Gatidis, Sergios and Clifford, Namuun and Daws, Joseph
+        and Tehrani, Arash and Kim, Jangwon and Chaudhari, Akshay},
+        journal={Journal of the American Medical Informatics Association},
+        volume={32},
+        number={3},
+        pages={470--479},
+        year={2024},
+        publisher={Oxford University Press}
+    }
+
+    @article{aali2024mimic,
+        title={MIMIC-IV-Ext-BHC: Labeled Clinical Notes Dataset for Hospital Course Summarization},
+        author={Aali, Asad and Van Veen, Dave and Arefeen, YI and Hom, Jason and Bluethgen, Christian
+        and Reis, Eduardo Pontes and Gatidis, Sergios and Clifford, Namuun and Daws, Joseph
+        and Tehrani, Arash and Kim, Jangwon and Chaudhari, Akshay},
+        journal={PhysioNet},
+        year={2024}
+    }
+    """
+
+    name = "mimic_bhc"
+    description = (
+        "A summarization task using a curated collection of preprocessed discharge notes"
+        " paired with their corresponding brief hospital course (BHC) summaries."
+    )
+    tags = ["summarization", "biomedical"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/share/pi/nigam/data/bhc-mimiciv/"
+        ensure_directory_exists(data_path)
+        data_path = data_path + "mimic_iv_bhc.json"
+
+        instances: List[Instance] = []
+        # Limit to zero shot setting for now
+        splits: Dict[str, str] = {
+            # "train": TRAIN_SPLIT,
+            # "validate": VALID_SPLIT,
+            "test": TEST_SPLIT,
+        }
+
+        with open(data_path, "r") as f:
+            data = [json.loads(line) for line in f]
+
+        for data_split, split in splits.items():
+            clinical_notes: List[str] = [x["input"] for x in data]
+            bhc_summaries: List[str] = [x["target"] for x in data]
+            assert len(clinical_notes) == len(bhc_summaries), "Notes and summaries must have the same length"
+            for clinical_note, bhc_summary in zip(clinical_notes, bhc_summaries):
+                if not clinical_note or not bhc_summary:
+                    continue
+                instances.append(
+                    Instance(
+                        input=Input(text=clinical_note),
+                        references=[Reference(Output(text=bhc_summary), tags=[CORRECT_TAG])],
+                        split=split,
+                    )
+                )
+
+        return instances

helm/benchmark/scenarios/mimic_rrs_scenario.py

@@ -0,0 +1,89 @@
+import os
+from typing import Dict, List
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+
+class MIMICRRSScenario(Scenario):
+    """
+    MIMIC-RRS is a biomedical question answering (QA) dataset collected from MIMIC-III and MIMIC-CXR
+    radiology reports.
+    In this scenario, we only consider the radiology reports from MIMIC-III.
+    In total, the dataset contains 73,259 reports.
+    The splits are provided by the dataset itself.
+
+    Sample Synthetic Prompt:
+        Generate the impressions of a radiology report based on its findings.
+
+        Findings:
+        The heart is normal in size. The lungs are clear.
+
+        Impressions:
+
+    @inproceedings{Chen_2023,
+        title={Toward Expanding the Scope of Radiology Report Summarization to Multiple Anatomies and Modalities},
+        url={http://dx.doi.org/10.18653/v1/2023.acl-short.41},
+        DOI={10.18653/v1/2023.acl-short.41},
+        booktitle={Proceedings of the 61st Annual Meeting of the Association
+                   for Computational Linguistics (Volume 2: Short Papers)},
+        publisher={Association for Computational Linguistics},
+        author={Chen, Zhihong and Varma, Maya and Wan, Xiang and Langlotz, Curtis and Delbrouck, Jean-Benoit},
+        year={2023},
+        pages={469–484}
+    }
+    """
+
+    name = "mimic_rrs"
+    description = (
+        "A dataset containing radiology reports with findings sections from MIMIC-III paired with"
+        " their corresponding impression sections, used for generating radiology report summaries."
+    )
+    tags = ["question_answering", "biomedical"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = "/share/pi/nigam/data/rrs-mimiciii/all"
+        ensure_directory_exists(data_path)
+
+        instances: List[Instance] = []
+        # Limit to zero shot setting for now
+        splits: Dict[str, str] = {
+            # "train": TRAIN_SPLIT,
+            # "validate": VALID_SPLIT,
+            "test": TEST_SPLIT,
+        }
+
+        for data_split, split in splits.items():
+            split_findings_name: str = f"{data_split}.findings.tok"
+            split_impressions_name: str = f"{data_split}.impression.tok"
+            findings_path: str = os.path.join(data_path, split_findings_name)
+            impressions_path: str = os.path.join(data_path, split_impressions_name)
+            findings: List[str] = self.read_file(findings_path)
+            impressions: List[str] = self.read_file(impressions_path)
+            assert len(findings) == len(impressions), "Findings and impressions must have the same length"
+            for finding, impression in zip(findings, impressions):
+                if not finding or not impression:
+                    continue
+                instances.append(
+                    Instance(
+                        input=Input(text=finding),
+                        references=[Reference(Output(text=impression), tags=[CORRECT_TAG])],
+                        split=split,
+                    )
+                )
+
+        return instances
+
+    def read_file(self, file_path: str) -> List[str]:
+        with open(file_path, "r") as file:
+            lines = file.readlines()
+        lines = [line.strip() for line in lines]
+        return lines

helm/benchmark/scenarios/mimiciv_billing_code_scenario.py

@@ -0,0 +1,71 @@
+import os
+import pandas as pd
+import numpy as np
+from typing import List
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+)
+
+
+class MIMICIVBillingCodeScenario(Scenario):
+    """
+    A scenario for MIMIC-IV discharge summaries where the task is to predict the ICD-10 code(s).
+
+    - Input:  The clinical note (column "text").
+    - Output: The list of ICD-10 codes (column "target").
+    """
+
+    name = "mimiciv_billing_code"
+    description = "A  dataset pairing clinical notes from MIMIC-IV with corresponding ICD-10 billing codes."
+    tags = ["question_answering", "biomedical"]
+
+    def __init__(self, data_file: str):
+        """
+        :param data_file: Path to the mimiciv_icd10.feather file.
+        """
+        super().__init__()
+        self.data_file = data_file
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        ensure_directory_exists(os.path.dirname(self.data_file))
+
+        # Read the preprocessed MIMIC-IV data (.feather format)
+        df = pd.read_feather(self.data_file)  # columns: ["text", "target", ...]
+
+        instances: List[Instance] = []
+
+        # Use the entire dataset as one split (TEST_SPLIT)
+        for idx, row in df.iterrows():
+            try:
+                note_text: str = row["text"]
+                icd10_codes = row["target"]
+
+                # Convert numpy array to list if necessary
+                if isinstance(icd10_codes, np.ndarray):
+                    icd10_codes = icd10_codes.tolist()
+                elif not isinstance(icd10_codes, list):
+                    icd10_codes = [str(icd10_codes)]  # Handle single values
+
+                # Convert all codes to strings and join
+                codes_str = ",".join(str(code) for code in icd10_codes)
+
+                # Create one Instance per row
+                instance = Instance(
+                    input=Input(text=note_text),
+                    references=[Reference(Output(text=codes_str), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+                instances.append(instance)
+            except Exception as e:
+                print(f"Error processing row {idx}: {str(e)}")
+                continue
+
+        return instances

helm/benchmark/scenarios/mmlu_clinical_afr_scenario.py

@@ -0,0 +1,74 @@
+import csv
+import os
+from typing import Dict, List
+
+from helm.common.general import ensure_file_downloaded
+from helm.common.hierarchical_logger import hlog
+from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+
+
+class MMLU_Clinical_Afr_Scenario(Scenario):
+    """
+    https://github.com/InstituteforDiseaseModeling/Bridging-the-Gap-Low-Resource-African-Languages
+    """
+
+    name = "mmlu_clinical_afr"
+    description = "Massive Multitask Language Understanding (MMLU) translated into 11 African low-resource languages"
+    tags = ["knowledge", "multiple_choice", "low_resource_languages"]
+
+    def __init__(self, subject: str = "clinical_knowledge", lang: str = "af"):
+        super().__init__()
+        self.subject: str = subject
+        self.lang: str = lang
+
+    def download_mmlu_clinical_afr(self, path: str):
+        ensure_file_downloaded(
+            source_url="https://github.com/InstituteforDiseaseModeling/Bridging-the-Gap-Low-Resource-African-Languages/raw/refs/heads/main/data/evaluation_benchmarks_afr_release.zip",  # noqa: E501
+            target_path=path,
+            unpack=True,
+            unpack_type="unzip",
+        )
+
+    def process_csv(self, csv_path: str, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        hlog(f"Reading {csv_path}")
+        with open(csv_path) as f:
+            reader = csv.reader(f, delimiter=",")
+            for row in reader:
+
+                question, answers, correct_choice = row[0], row[1:-1], row[-1]
+                answers_dict = dict(zip(["A", "B", "C", "D"], answers))
+                correct_answer: str = answers_dict[correct_choice]
+
+                def answer_to_reference(answer: str) -> Reference:
+                    return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+                instance = Instance(
+                    input=Input(text=question),
+                    references=list(map(answer_to_reference, answers)),
+                    split=split,
+                )
+                instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data
+        desired_dir = "mmlu_cm_ck_vir"
+        data_path: str = os.path.join(output_path, desired_dir)
+        self.download_mmlu_clinical_afr(data_path)
+
+        # Read all the instances
+        instances: List[Instance] = []
+        splits: Dict[str, str] = {
+            "dev": TRAIN_SPLIT,
+            "val": VALID_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for split in splits:
+            csv_path: str = os.path.join(data_path, desired_dir, f"{self.subject}_{split}_{self.lang}.csv")
+            if not os.path.exists(csv_path):
+                hlog(f"{csv_path} doesn't exist, skipping")
+                continue
+            instances.extend(self.process_csv(csv_path, splits[split]))
+
+        return instances

helm/benchmark/scenarios/mmlu_pro_scenario.py

@@ -0,0 +1,95 @@
+from typing import Dict, List
+from datasets import Dataset, load_dataset
+
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MMLUProScenario(Scenario):
+    """
+    The MMLU-Pro dataset is an advanced version of the Massive Multitask Language Understanding (MMLU)
+    benchmark, created to push the boundaries of language models' reasoning and comprehension skills.
+    Designed as a more challenging evaluation, it increases the answer options per question from four
+    to ten, significantly reducing the likelihood of correct random guesses. This update makes the
+    dataset better at distinguishing the capabilities of models on complex tasks.
+
+    MMLU-Pro emphasizes reasoning over simple factual recall by integrating diverse, intricate questions
+    across 14 domains, including subjects like biology, economics, law, and psychology. In addition, it
+    addresses limitations in the original MMLU by filtering out trivial questions, making it a more
+    robust benchmark. Performance comparisons suggest that models benefit from reasoning-based
+    approaches (such as Chain of Thought, or CoT) on MMLU-Pro, which contrasts with the original
+    MMLU where CoT didn’t show as much benefit. This makes MMLU-Pro especially suitable for evaluating
+    advanced models that rely on nuanced reasoning and comprehension skills​.
+
+    Dataset: https://huggingface.co/datasets/TIGER-Lab/MMLU-Pro
+    Paper: https://arxiv.org/abs/2406.01574
+    """
+
+    name = "mmlu_pro"
+    description = "Enhanced Massive Multitask Language Understanding with increased options and reasoning"
+    tags = ["knowledge", "multiple_choice", "reasoning"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        self.subject: str = subject
+
+    def process_dataset(self, data: Dataset, split: str) -> List[Instance]:
+        """
+        Process the dataset to create instances.
+
+        :param data: Hugging Face `Dataset` containing the data for a specific split.
+        :param split: The data split (e.g., "train", "test").
+        :return: A list of processed `Instance` objects.
+        """
+        instances: List[Instance] = []
+        hlog(f"Processing data for {split} split")
+        for row in data:
+            id = row["question_id"]
+            question = row["question"]
+            answers = row["options"]
+            correct_choice = row["answer"]
+            answers_dict = dict(zip(["A", "B", "C", "D", "E", "F", "G", "H", "I", "J"], answers))
+            correct_answer = answers_dict[correct_choice]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+
+            instance = Instance(
+                id=f"id{id}",
+                input=Input(text=question),
+                references=list(map(answer_to_reference, answers)),
+                split=split,
+            )
+            instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Load and process the MMLU-Pro dataset to create instances.
+
+        :param output_path: Path to save or output the processed instances.
+        :return: A list of all processed `Instance` objects.
+        """
+        # Load the MMLU-Pro dataset from Hugging Face
+        dataset = load_dataset("TIGER-Lab/MMLU-Pro", revision="3373e0b")
+
+        # Process all the instances
+        instances: List[Instance] = []
+        splits: Dict[str, str] = {
+            "validation": TRAIN_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for hf_split, split in splits.items():
+            data = dataset[hf_split].filter(lambda x: self.subject == "all" or self.subject == x["category"])
+            instances.extend(self.process_dataset(data, split))
+
+        return instances

helm/benchmark/scenarios/mmlu_scenario.py

@@ -4,7 +4,17 @@ from typing import Dict, List
 
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
 
 
 class MMLUScenario(Scenario):

helm/benchmark/scenarios/msmarco_scenario.py

@@ -6,7 +6,7 @@ from typing import Dict, List, Tuple, Optional, Union
 
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.hierarchical_logger import hlog
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,