crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/melt_translation_scenario.py

@@ -0,0 +1,152 @@
+from typing import Any, Dict, List, Optional
+
+from datasets import load_dataset, Dataset
+from helm.common.hierarchical_logger import htrack_block
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MELTTranslationScenario(Scenario):
+    name = "melt_translation"
+    description = "Machine Translation scenario."
+    tags = ["machine_translation"]
+
+    def __init__(
+        self,
+        dataset_name: str,
+        revision: str,
+        source_language: str,
+        target_language: str,
+        subset: Optional[str] = None,
+        splits: Optional[Dict[str, str]] = None,
+    ):
+        """Initializes the question answering scenario.
+
+        Args:
+            dataset_name: The name of the dataset.
+            revision: The revision of the dataset to use.
+            source_language: The source language to use.
+            target_language: The target language to use.
+            subset: The subset of the dataset to use. Defaults to "".
+            splits: The splits to use for the dataset. Defaults to None.
+        """
+        super().__init__()
+        self.MAX_TRAIN_INSTANCES = 20_000
+        valid_languages = set(["vi", "en"])
+        self.dataset_name = dataset_name
+        self.subset = subset
+        self.revision = revision
+        self.splits = splits
+        self.source_language = source_language
+        self.target_language = target_language
+        if self.source_language not in valid_languages or self.target_language not in valid_languages:
+            raise ValueError("Supported languages: vi, en.")
+        if self.source_language == self.target_language:
+            raise ValueError("The source language and the target language should be different.")
+        if self.source_language != "en" and self.target_language != "en":
+            raise ValueError("One of the languages should be English.")
+
+    def get_instances_for_splits(self, splits: Dict[str, str]) -> List[Instance]:
+        """
+        Helper for generating instances for a split.
+        Args:
+            splits (dict): Which splits to partition the data into.
+        Returns:
+            List[Instance]: Instances from the file for the specified split.
+        """
+        with htrack_block("Loading the HuggingFace dataset. The first time could take several minutes."):
+            hf_dataset: Any = load_dataset(
+                self.dataset_name,
+                self.subset,
+                revision=self.revision,
+                trust_remote_code=True,
+            )
+
+        instances: List[Instance] = []
+
+        for dataset_split_name, helm_split_name in splits.items():
+            if helm_split_name == TRAIN_SPLIT:
+                hf_dataset[dataset_split_name] = hf_dataset[dataset_split_name].shuffle(seed=42)[
+                    : self.MAX_TRAIN_INSTANCES
+                ]
+                hf_dataset[dataset_split_name] = Dataset.from_dict(hf_dataset[dataset_split_name])
+
+            for example in hf_dataset[dataset_split_name]:
+                source_sentence = example[self.source_language]
+                target_sentence = example[self.target_language]
+                instances.append(
+                    Instance(
+                        input=Input(text=source_sentence),
+                        references=[Reference(Output(text=target_sentence), tags=[CORRECT_TAG])],
+                        split=helm_split_name,
+                    )
+                )
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        if self.splits is None:
+            splits = {"train": TRAIN_SPLIT, "validation": VALID_SPLIT, "test": TEST_SPLIT}
+        else:
+            splits = {}
+            if "train" in self.splits:
+                splits[self.splits[TRAIN_SPLIT]] = TRAIN_SPLIT
+            if "validation" in self.splits:
+                splits[self.splits[VALID_SPLIT]] = VALID_SPLIT
+            if "test" in self.splits:
+                splits[self.splits[TEST_SPLIT]] = TEST_SPLIT
+
+        instances: List[Instance] = self.get_instances_for_splits(splits=splits)
+        return instances
+
+
+class MELTTranslationOPUS100Scenario(MELTTranslationScenario):
+    """
+    Scenario for the OPUS100 dataset.
+    """
+
+    name = "melt_translation_opus100"
+    description = "OPUS100 dataset for machine translation."
+    tags = ["machine_translation"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="vietgpt/opus100_envi",
+            revision="45df06fb0b31edc882d7c8d34389261f995e5208",
+            splits={
+                TRAIN_SPLIT: "train",
+                VALID_SPLIT: "validation",
+                TEST_SPLIT: "test",
+            },
+            **kwargs,
+        )
+
+
+class MELTTranslationPhoMTScenario(MELTTranslationScenario):
+    """
+    Scenario for the PhoMT dataset.
+    """
+
+    name = "melt_translation_phomt"
+    description = "PhoMT dataset for machine translation."
+    tags = ["machine_translation"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="ura-hcmut/PhoMT",
+            revision="74386685db01dc038860ff0a90d9f5fbde284bf7",
+            splits={
+                TRAIN_SPLIT: "train",
+                VALID_SPLIT: "validation",
+                TEST_SPLIT: "test",
+            },
+            **kwargs,
+        )

helm/benchmark/scenarios/mental_health_scenario.py

@@ -0,0 +1,146 @@
+import pandas as pd
+from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    PassageQuestionInput,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import check_file_exists
+
+
+class MentalHealthScenario(Scenario):
+    """
+    This scenario evaluates language models' ability to generate appropriate counseling responses
+    in mental health conversations. The dataset contains counseling dialogues covering
+    various topics including workplace issues, anxiety, suicidal thoughts, relationship
+    problems, and more.
+
+    Each dialogue consists of interactions between a counselor and a client, where the counselor
+    demonstrates expert mental health counseling techniques. The dialogues were selected based on high
+    quality scores from multiple evaluators.
+
+    Example dialogue structure:
+    ```
+    counselor: Hi there, to start can you tell me your name and a little bit about what's been going on?
+    client: I sleep too much... I'm 23, female and work as IT professional. I feel like I'm not fitting in...
+    counselor: I can see you have been facing challenges with feeling like you don't fit in...
+    ```
+
+    The task is to generate the next counselor response given the conversation history. Models
+    are evaluated on their ability to:
+    1. Provide empathetic and supportive responses
+    2. Follow proper mental health counseling protocols
+    3. Generate contextually appropriate interventions
+
+    The dataset includes:
+    - 7 complete dialogues covering different mental health topics
+    - Metadata about dialogue topic and type
+    - Gold-standard counselor responses as references
+    - Full conversation history for context
+
+    Each instance includes:
+    - input: Previous conversation turns formatted with speaker labels
+    - reference: The actual counselor's response (gold standard)
+    - metadata: Topic and type of mental health conversation
+    """
+
+    name = "mental_health"
+    description = (
+        "MentalHealth is a benchmark focused on evaluating empathetic communication in"
+        "mental health counseling. It includes simulated conversations between patients"
+        "and counselors, where the task is to generate compassionate and appropriate counselor"
+        "responses. The benchmark assesses a model's ability to support patients emotionally"
+        "and meaningfully engage in therapeutic conversations."
+    )
+    tags = ["dialogue", "counseling", "mental_health", "empathy", "healthcare"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def process_dialogue_data(self, data: pd.DataFrame) -> List[Instance]:
+        """
+        Process the dialogue data into evaluation instances.
+        Each instance represents a point in the conversation where the model needs to generate
+        a counselor response.
+
+        Args:
+            data (pd.DataFrame): DataFrame containing processed dialogues with columns:
+                - context: Previous conversation history
+                - gold_counselor_response: Reference counselor response
+                - topic: Type of conversation (e.g., "Anxiety", "Workplace")
+                - dialogue_type: Numerical type identifier
+
+        Returns:
+            List[Instance]: List of processed instances ready for evaluation
+        """
+        instances: List[Instance] = []
+
+        for _, row in data.iterrows():
+            # Format input with clear section breaks and instructions
+            input_text = (
+                f"Topic: {row['topic']}\n"
+                f"Type: {row['dialogue_type']}\n"
+                f"\nPrevious conversation:\n{row['context']}\n"
+                f"\nGenerate an empathetic and appropriate counselor response:"
+            )
+
+            # Create input with empty passage since all context is in question
+            prompt = PassageQuestionInput(passage="", question=input_text)
+
+            # Create instance with gold standard response
+            instance = Instance(
+                input=prompt,
+                references=[Reference(Output(text=row["gold_counselor_response"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """
+        Load and process the mental health dialogue dataset.
+
+        Args:
+            output_path (str): Path for any cached or intermediate files
+
+        Returns:
+            List[Instance]: List of processed instances for evaluation
+        """
+        check_file_exists(
+            self.data_path, msg=f"[MentalHealthScenario] Required data file not found: '{self.data_path}'"
+        )
+        dialogue_data = pd.read_csv(self.data_path)
+
+        # Process into instances
+        instances = self.process_dialogue_data(dialogue_data)
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mental_health",
+            display_name="MentalHealth",
+            description="MentalHealth is a benchmark focused on evaluating empathetic communication in "
+            "mental health counseling. It includes real or simulated conversations between "
+            "patients and counselors, where the task is to generate compassionate and "
+            "appropriate counselor responses. The benchmark assesses a model's ability to "
+            "support patients emotionally and meaningfully engage in therapeutic "
+            "conversations.",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate empathetic counseling responses in mental health " "conversations",
+                when="Any",
+                who="Counselors, Patients",
+                language="English",
+            ),
+            main_metric="mental_health_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/mimic_bhc_scenario.py

@@ -0,0 +1,127 @@
+import json
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import check_file_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+
+
+class MIMICBHCScenario(Scenario):
+    r"""
+    MIMIC-IV-BHC presents a curated collection of preprocessed discharge notes with labeled brief hospital
+    course (BHC) summaries. This dataset is derived from MIMIC-IV (https://doi.org/10.1093/jamia/ocae312).
+
+    In total, the dataset contains 270,033 clinical notes.
+    The splits are provided by the dataset itself.
+
+    Sample Synthetic Prompt:
+        Summarize the clinical note into a brief hospital course.
+
+        Clinical Note:
+        <SEX> M <SERVICE> SURGERY <ALLERGIES> No Known Allergies \/ Adverse Drug Reactions
+        ...
+        continue to follow-up with your health care providers as an outpatient.
+
+        Brief Hospital Course:
+        Mr. ___ was pre-admitted on ___ for liver transplantation
+        ...
+        discharged home to continue home medications and follow-up as an outpatient.
+
+    @article{aali2024dataset,
+        title={A dataset and benchmark for hospital course summarization with adapted large language models},
+        author={Aali, Asad and Van Veen, Dave and Arefeen, YI and Hom, Jason and Bluethgen, Christian
+        and Reis, Eduardo Pontes and Gatidis, Sergios and Clifford, Namuun and Daws, Joseph
+        and Tehrani, Arash and Kim, Jangwon and Chaudhari, Akshay},
+        journal={Journal of the American Medical Informatics Association},
+        volume={32},
+        number={3},
+        pages={470--479},
+        year={2024},
+        publisher={Oxford University Press}
+    }
+
+    @article{aali2024mimic,
+        title={MIMIC-IV-Ext-BHC: Labeled Clinical Notes Dataset for Hospital Course Summarization},
+        author={Aali, Asad and Van Veen, Dave and Arefeen, YI and Hom, Jason and Bluethgen, Christian
+        and Reis, Eduardo Pontes and Gatidis, Sergios and Clifford, Namuun and Daws, Joseph
+        and Tehrani, Arash and Kim, Jangwon and Chaudhari, Akshay},
+        journal={PhysioNet},
+        year={2024}
+    }
+    """
+
+    name = "mimic_bhc"
+    description = (
+        "MIMIC-BHC is a benchmark focused on summarization of discharge notes into Brief"
+        "Hospital Course (BHC) sections. It consists of curated discharge notes from MIMIC-IV,"
+        "each paired with its corresponding BHC summary. The benchmark evaluates a model's"
+        "ability to condense detailed clinical information into accurate, concise summaries that"
+        "reflect the patient's hospital stay."
+    )
+    tags = ["summarization", "biomedical"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.data_path, msg=f"[MIMICBHCScenario] Required data file not found: '{self.data_path}'")
+
+        instances: List[Instance] = []
+        # Limit to zero shot setting for now
+        splits: Dict[str, str] = {
+            # "train": TRAIN_SPLIT,
+            # "validate": VALID_SPLIT,
+            "test": TEST_SPLIT,
+        }
+
+        with open(self.data_path, "r") as f:
+            data = [json.loads(line) for line in f]
+
+        for data_split, split in splits.items():
+            clinical_notes: List[str] = [x["input"] for x in data]
+            bhc_summaries: List[str] = [x["target"] for x in data]
+            assert len(clinical_notes) == len(bhc_summaries), "Notes and summaries must have the same length"
+            for clinical_note, bhc_summary in zip(clinical_notes, bhc_summaries):
+                if not clinical_note or not bhc_summary:
+                    continue
+                instances.append(
+                    Instance(
+                        input=Input(text=clinical_note),
+                        references=[Reference(Output(text=bhc_summary), tags=[CORRECT_TAG])],
+                        split=split,
+                    )
+                )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mimic_bhc",
+            display_name="MIMIC-IV-BHC",
+            short_display_name="MIMIC-BHC",
+            description="MIMIC-BHC is a benchmark focused on summarization of discharge notes into "
+            "Brief Hospital Course (BHC) sections. It consists of curated discharge notes "
+            "from MIMIC-IV, each paired with its corresponding BHC summary. The benchmark "
+            "evaluates a model's ability to condense detailed clinical information into "
+            "accurate, concise summaries that reflect the patient's hospital stay [(Aali et "
+            "al., 2024)](https://doi.org/10.1093/jamia/ocae312).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Summarize the clinical note into a brief hospital course",
+                when="Upon hospital discharge",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="mimic_bhc_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/mimic_rrs_scenario.py

@@ -0,0 +1,121 @@
+import os
+from typing import Dict, List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import check_file_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+
+
+class MIMICRRSScenario(Scenario):
+    """
+    MIMIC-RRS is a biomedical question answering (QA) dataset collected from MIMIC-III and MIMIC-CXR
+    radiology reports.
+    In this scenario, we only consider the radiology reports from MIMIC-III.
+    In total, the dataset contains 73,259 reports.
+    The splits are provided by the dataset itself.
+
+    Sample Synthetic Prompt:
+        Generate the impressions of a radiology report based on its findings.
+
+        Findings:
+        The heart is normal in size. The lungs are clear.
+
+        Impressions:
+
+    @inproceedings{Chen_2023,
+        title={Toward Expanding the Scope of Radiology Report Summarization to Multiple Anatomies and Modalities},
+        url={http://dx.doi.org/10.18653/v1/2023.acl-short.41},
+        DOI={10.18653/v1/2023.acl-short.41},
+        booktitle={Proceedings of the 61st Annual Meeting of the Association
+                   for Computational Linguistics (Volume 2: Short Papers)},
+        publisher={Association for Computational Linguistics},
+        author={Chen, Zhihong and Varma, Maya and Wan, Xiang and Langlotz, Curtis and Delbrouck, Jean-Benoit},
+        year={2023},
+        pages={469–484}
+    }
+    """
+
+    name = "mimic_rrs"
+    description = (
+        "MIMIC-RRS is a benchmark constructed from radiology reports in the MIMIC-III"
+        "database. It contains pairs of 'Findings' and 'Impression' sections, enabling evaluation"
+        "of a model's ability to summarize diagnostic imaging observations into concise, clinically"
+        "relevant conclusions."
+    )
+    tags = ["question_answering", "biomedical"]
+
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        instances: List[Instance] = []
+        # Limit to zero shot setting for now
+        splits: Dict[str, str] = {
+            # "train": TRAIN_SPLIT,
+            # "validate": VALID_SPLIT,
+            "test": TEST_SPLIT,
+        }
+
+        for data_split, split in splits.items():
+            split_findings_name: str = f"{data_split}.findings.tok"
+            split_impressions_name: str = f"{data_split}.impression.tok"
+            findings_path: str = os.path.join(self.data_path, split_findings_name)
+            impressions_path: str = os.path.join(self.data_path, split_impressions_name)
+            check_file_exists(
+                findings_path, msg=f"[MIMICRRSScenario] Required findings file not found: '{findings_path}'"
+            )
+            check_file_exists(
+                impressions_path, msg=f"[MIMICRRSScenario] Required impressions file not found: '{impressions_path}'"
+            )
+            findings: List[str] = self.read_file(findings_path)
+            impressions: List[str] = self.read_file(impressions_path)
+            assert len(findings) == len(impressions), "Findings and impressions must have the same length"
+            for finding, impression in zip(findings, impressions):
+                if not finding or not impression:
+                    continue
+                instances.append(
+                    Instance(
+                        input=Input(text=finding),
+                        references=[Reference(Output(text=impression), tags=[CORRECT_TAG])],
+                        split=split,
+                    )
+                )
+
+        return instances
+
+    def read_file(self, file_path: str) -> List[str]:
+        with open(file_path, "r") as file:
+            lines = file.readlines()
+        lines = [line.strip() for line in lines]
+        return lines
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mimic_rrs",
+            display_name="MIMIC-RRS",
+            short_display_name="MIMIC-RRS",
+            description="MIMIC-RRS is a benchmark constructed from radiology reports in the MIMIC-III "
+            "database. It contains pairs of ‘Findings‘ and ‘Impression‘ sections, enabling "
+            "evaluation of a model's ability to summarize diagnostic imaging observations "
+            "into concise, clinically relevant conclusions [(Chen et al., "
+            "2023)](https://arxiv.org/abs/2211.08584).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate radiology report summaries from findings sections",
+                when="Post-imaging",
+                who="Radiologist",
+                language="English",
+            ),
+            main_metric="mimic_rrs_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/mimiciv_billing_code_scenario.py

@@ -0,0 +1,99 @@
+import pandas as pd
+import numpy as np
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import check_file_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+
+
+class MIMICIVBillingCodeScenario(Scenario):
+    """
+    A scenario for MIMIC-IV discharge summaries where the task is to predict the ICD-10 code(s).
+
+    - Input:  The clinical note (column "text").
+    - Output: The list of ICD-10 codes (column "target").
+    """
+
+    name = "mimiciv_billing_code"
+    description = (
+        "MIMIC-IV Billing Code is a benchmark derived from discharge summaries in the"
+        "MIMIC-IV database, paired with their corresponding ICD-10 billing codes. The task"
+        "requires models to extract structured billing codes based on free-text clinical notes,"
+        "reflecting real-world hospital coding tasks for financial reimbursement."
+    )
+    tags = ["question_answering", "biomedical"]
+
+    def __init__(self, data_path: str):
+        """
+        :param data_path: Path to the mimiciv_icd10.feather file.
+        """
+        super().__init__()
+        self.data_path = data_path
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(
+            self.data_path, msg=f"[MIMICIVBilligCodeScenario] Required data file not found: '{self.data_path}'"
+        )
+
+        # Read the preprocessed MIMIC-IV data (.feather format)
+        df = pd.read_feather(self.data_path)  # columns: ["text", "target", ...]
+
+        instances: List[Instance] = []
+
+        # Use the entire dataset as one split (TEST_SPLIT)
+        for idx, row in df.iterrows():
+            try:
+                note_text: str = row["text"]
+                icd10_codes = row["target"]
+
+                # Convert numpy array to list if necessary
+                if isinstance(icd10_codes, np.ndarray):
+                    icd10_codes = icd10_codes.tolist()
+                elif not isinstance(icd10_codes, list):
+                    icd10_codes = [str(icd10_codes)]  # Handle single values
+
+                # Convert all codes to strings and join
+                codes_str = ",".join(str(code) for code in icd10_codes)
+
+                # Create one Instance per row
+                instance = Instance(
+                    input=Input(text=note_text),
+                    references=[Reference(Output(text=codes_str), tags=[CORRECT_TAG])],
+                    split=TEST_SPLIT,
+                )
+                instances.append(instance)
+            except Exception as e:
+                print(f"Error processing row {idx}: {str(e)}")
+                continue
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mimiciv_billing_code",
+            display_name="MIMIC-IV Billing Code",
+            description="MIMIC-IV Billing Code is a benchmark derived from discharge summaries in the "
+            "MIMIC-IV database, paired with their corresponding ICD-10 billing codes. The "
+            "task requires models to extract structured billing codes based on free-text "
+            "clinical notes, reflecting real-world hospital coding tasks for financial "
+            "reimbursement.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Predict ICD-10 billing codes from clinical discharge notes",
+                when="During or after patient discharge",
+                who="Hospital Admistrator",
+                language="English",
+            ),
+            main_metric="mimiciv_billing_code_f1",
+            main_split="test",
+        )