crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/medec_scenario.py

@@ -0,0 +1,148 @@
+import csv
+import os
+from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_file_downloaded
+
+
+class MedecScenario(Scenario):
+    """
+    Processes the MEDEC dataset for medical error detection and correction tasks.
+
+    MEDEC is the first publicly available benchmark for medical error detection and correction in clinical notes,
+    introduced in "Ben Abacha et al., 2024." The dataset includes 3,848 clinical texts from the MS and UW collections,
+    covering five types of errors:
+    - Diagnosis
+    - Management
+    - Treatment
+    - Pharmacotherapy
+    - Causal Organism
+
+    The dataset consists of:
+    - Training Set: 2,189 MS texts
+    - Validation Set: 574 MS texts and 160 UW texts
+    - Test Set: 597 MS texts and 328 UW texts
+
+    Each clinical text is labeled as either correct or containing one error. The task involves:
+    (A) Predicting the error flag (1: the text contains an error, 0: the text has no errors).
+    (B) For flagged texts, extracting the sentence that contains the error.
+    (C) Generating a corrected sentence.
+
+    The MEDEC dataset was used for the MEDIQA-CORR shared task to evaluate seventeen participating systems.
+    Recent LLMs (e.g., GPT-4, Claude 3.5 Sonnet, Gemini 2.0 Flash) have been evaluated on this dataset, showing good
+    performance but still lagging behind medical doctors in error detection and correction tasks.
+
+    Task:
+    Given a clinical text, models must identify errors and correct them while demonstrating medical knowledge
+    and reasoning capabilities.
+    """
+
+    GIT_HASH = "49c59dcba43a7af8717590a405b11a57f42b04df"
+    TRAIN_URL = f"https://raw.githubusercontent.com/abachaa/MEDEC/{GIT_HASH}/MEDEC-MS/MEDEC-Full-TrainingSet-with-ErrorType.csv"  # noqa: E501
+    TEST_URL = f"https://raw.githubusercontent.com/abachaa/MEDEC/{GIT_HASH}/MEDEC-MS/MEDEC-MS-TestSet-with-GroundTruth-and-ErrorType.csv"  # noqa: E501
+
+    name = "medec"
+    description = (
+        "Medec is a benchmark composed of clinical narratives that include either correct"
+        "documentation or medical errors. Each entry includes sentence-level identifiers and an"
+        "associated correction task. The model must review the narrative and either identify"
+        "the erroneous sentence and correct it, or confirm that the text is entirely accurate."
+    )
+    tags = ["error_detection", "error_correction", "biomedical"]
+
+    def download_csv(self, url: str, output_path: str, file_name: str) -> str:
+        """Download the CSV file and return its path."""
+        csv_path = os.path.join(output_path, file_name)
+        ensure_file_downloaded(source_url=url, target_path=csv_path, unpack=False)
+        return csv_path
+
+    def process_csv(self, csv_path: str, split: str) -> List[Instance]:
+        """Read and process a CSV file to generate instances."""
+        instances: List[Instance] = []
+        with open(csv_path, "r", encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+
+            for row in reader:
+                # Ensure required fields exist
+                if not row.get("Sentences"):
+                    continue
+
+                # Extract text content
+                input_text = row["Sentences"].strip()
+
+                # Extract metadata
+                error_flag = int(row.get("Error Flag", 0))
+                error_sentence_id = row.get("Error Sentence ID", "-1").strip()
+                corrected_sentence = row.get("Corrected Sentence", "").strip()
+
+                references = []
+                if error_flag == 1 and corrected_sentence != "NA" and error_sentence_id != "-1":
+                    # If there is an error, store the corrected sentence
+                    references.append(
+                        Reference(
+                            Output(text=f"{error_sentence_id} {corrected_sentence}"),
+                            tags=[CORRECT_TAG],
+                        )
+                    )
+                else:
+                    # If no error exists, expect "CORRECT" as the output
+                    references.append(
+                        Reference(
+                            Output(text="CORRECT"),
+                            tags=[CORRECT_TAG],
+                        )
+                    )
+
+                instance = Instance(
+                    input=Input(text=input_text),
+                    references=references,
+                    split=split,
+                )
+                instances.append(instance)
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Download and process the dataset to generate instances."""
+        instances: List[Instance] = []
+
+        # Download and process the training set - limit to zero shot setting for now
+        # train_csv = self.download_csv(self.TRAIN_URL, output_path, "medec_train.csv")
+        # instances.extend(self.process_csv(train_csv, TRAIN_SPLIT))
+
+        # Download and process the test set
+        test_csv = self.download_csv(self.TEST_URL, output_path, "medec_test.csv")
+        instances.extend(self.process_csv(test_csv, TEST_SPLIT))
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medec",
+            display_name="Medec",
+            description="Medec is a benchmark composed of clinical narratives that include either "
+            "correct documentation or medical errors. Each entry includes sentence-level "
+            "identifiers and an associated correction task. The model must review the "
+            "narrative and either identify the erroneous sentence and correct it, or "
+            "confirm that the text is entirely accurate [(Abacha et al., "
+            "2025)](https://arxiv.org/abs/2412.19260).",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Detect and correct errors in medical narratives",
+                when="Any",
+                who="Researcher, Clinician",
+                language="English",
+            ),
+            main_metric="medec_error_flag_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/medhallu_scenario.py

@@ -0,0 +1,95 @@
+from typing import List
+from datasets import load_dataset
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Output,
+    Input,
+    ScenarioMetadata,
+)
+
+
+class MedHalluScenario(Scenario):
+    """
+    MedHallu is a medical hallucination dataset that consists of PubMed articles and associated questions,
+    with the objective being to classify whether the answer is factual or hallucinated.
+    MedHallu: https://medhallu.github.io/
+    """
+
+    name = "medhallu"
+    description = (
+        "MedHallu is a benchmark focused on evaluating factual correctness in biomedical"
+        "question answering. Each instance contains a PubMed-derived knowledge snippet, a"
+        "biomedical question, and a model-generated answer. The task is to classify whether the"
+        "answer is factually correct or contains hallucinated (non-grounded) information. This"
+        "benchmark is designed to assess the factual reliability of medical language models."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    def create_instance(self, question, knowledge, answer, label, split):
+        prompt_text = f"""World Knowledge: {knowledge}
+
+Question: {question}
+
+Answer: {answer}
+"""
+        return Instance(
+            input=Input(
+                text=prompt_text,
+            ),
+            references=[Reference(Output(text=label), tags=[CORRECT_TAG])],
+            split=split,
+        )
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Load the MedCalc-Bench dataset from Hugging Face
+        dataset = load_dataset(
+            "UTAustin-AIHealth/MedHallu",
+            "pqa_labeled",
+            split="train",
+            revision="515060458a945c633debc6fd5baac7764416b724",
+        )
+
+        # Process all the instances - limit to zero shot setting
+        instances: List[Instance] = []
+        for row in dataset:
+            # print("printing row", row)
+            question = row["Question"]
+            ground_truth_answer = row["Ground Truth"]
+            knowledge = row["Knowledge"]
+            hallucinated_answer = row["Hallucinated Answer"]
+
+            gt_instance = self.create_instance(question, knowledge, ground_truth_answer, label="0", split=TEST_SPLIT)
+            instances.append(gt_instance)
+
+            hallucinated_instance = self.create_instance(
+                question, knowledge, hallucinated_answer, label="1", split=TEST_SPLIT
+            )
+            instances.append(hallucinated_instance)
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medhallu",
+            display_name="MedHallu",
+            description="MedHallu is a benchmark focused on evaluating factual correctness in "
+            "biomedical question answering. Each instance contains a PubMed-derived "
+            "knowledge snippet, a biomedical question, and a model-generated answer. The "
+            "task is to classify whether the answer is factually correct or contains "
+            "hallucinated (non-grounded) information. This benchmark is designed to assess "
+            "the factual reliability of medical language models.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Verify whether answers to questions from PubMed articles are " "factual or hallucinated",
+                when="Any",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/medhelm/judges.yaml

@@ -0,0 +1,14 @@
+# The judges to be used for evaluating the note summary scenario.
+# name: The short name for the judge.
+# model: The field value matching the 'model_name' field under model_deployments.yaml
+# model_deployment: The field value matching the 'name' under model_deployments.yaml.
+judges:
+  - name: "gpt"
+    model: "openai/gpt-4o-2024-05-13"
+    model_deployment: "stanfordhealthcare/gpt-4o-2024-05-13"
+  - name: "llama"
+    model: "meta/llama-3.3-70b-instruct"
+    model_deployment: "stanfordhealthcare/llama-3.3-70b-instruct"
+  - name: "claude"
+    model: "anthropic/claude-3-7-sonnet-20250219"
+    model_deployment: "stanfordhealthcare/claude-3-7-sonnet-20250219"

helm/benchmark/scenarios/medhelm_configurable_scenario.py

@@ -0,0 +1,101 @@
+import string
+import json
+import pandas as pd
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.run_specs.medhelm.benchmark_config import get_benchmark_config_from_path
+from helm.common.general import check_file_exists
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    CORRECT_TAG,
+    Reference,
+    Input,
+    Output,
+    TEST_SPLIT,
+    ScenarioMetadata,
+)
+
+
+class MedHELMConfigurableScenario(Scenario):
+    """
+    MedHELM configuratble scenario
+    """
+
+    tags = ["biomedical"]
+
+    def __init__(self, name: str, config_path: str):
+        super().__init__()
+        self.benchmark_config = get_benchmark_config_from_path(config_path)
+        self.name = name
+        self.description = self.benchmark_config.description
+
+    def get_columns_in_template(self, template: str) -> List[str]:
+        """
+        Extract field names from a template string using Python's Formatter.
+        Example: "Name: {name}, Age: {age}" → ["name", "age"]
+        """
+        formatter = string.Formatter()
+        return [fname for _, fname, _, _ in formatter.parse(template) if fname]
+
+    def populate_template(self, template: str, row: pd.Series, fields: List[str]) -> str:
+        """
+        Populate the template with values from the row using format_map.
+        Missing fields default to empty string.
+        """
+        mapping = {field: row.get(field, "") for field in fields}
+        return template.format_map(mapping)
+
+    def get_references(self, row: pd.Series) -> List[Reference]:
+        references: List[Reference] = []
+        if "correct_answer" in row:
+            references.append(Reference(Output(text=row["correct_answer"]), tags=[CORRECT_TAG]))
+        if "incorrect_answers" in row:
+            for incorrect_answer in row["incorrect_answers"]:
+                references.append(Reference(Output(text=incorrect_answer), tags=[]))
+        return references
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.benchmark_config.prompt_file, msg=f"Prompt file for {self.name} does not exist")
+        check_file_exists(self.benchmark_config.dataset_file, msg=f"Dataset file for {self.name} does not exist")
+        instances: List[Instance] = []
+        df = pd.read_csv(self.benchmark_config.dataset_file)
+        if "correct_answer" not in df.columns:
+            if not self._is_llm_as_judge() or len(self.benchmark_config.metrics) > 1:
+                raise ValueError(
+                    "Dataset must contain 'correct_answer' column unless using jury_score as the only metric."
+                )
+        if "incorrect_answers" in df.columns:
+            df["incorrect_answers"] = df["incorrect_answers"].apply(json.loads)
+        with open(self.benchmark_config.prompt_file, "r") as f:
+            template = f.read()
+        fields = self.get_columns_in_template(template)
+        for _, row in df.iterrows():
+            filled = self.populate_template(template, row, fields)
+            prompt = Input(text=filled)
+            instances.append(Instance(input=prompt, references=self.get_references(row), split=TEST_SPLIT))
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name=self.name,
+            display_name=self.name,
+            description=self.description,
+            taxonomy=TaxonomyInfo(
+                task="",
+                what="",
+                when="",
+                who="",
+                language="",
+            ),
+            main_metric=self.benchmark_config.main_metric.name,
+            main_split="test",
+        )
+
+    def _is_llm_as_judge(self) -> bool:
+        for metric in self.benchmark_config.metrics:
+            if metric.name == "jury_score":
+                return True
+        return False

helm/benchmark/scenarios/medi_qa_scenario.py

@@ -0,0 +1,134 @@
+from typing import Dict, List
+from datasets import load_dataset
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+
+
+class MediQAScenario(Scenario):
+    """
+    MEDIQA-QA is a dataset designed to benchmark large language models (LLMs) on medical
+    question answering (QA) tasks.
+    Each instance in the dataset includes a medical question, a set of candidate answers,
+    relevance annotations for ranking, and additional context to evaluate understanding
+    and retrieval capabilities in a healthcare setting.
+
+    The dataset encompasses diverse question types, including consumer health queries
+    and clinical questions, making it suitable for assessing LLMs' ability to answer
+    consumer healthcare questions.
+
+    This dataset comprises two training sets of 104 instances each, a validation set
+    of 25 instances, and a testing set of 150 instances.
+
+    Dataset: https://huggingface.co/datasets/bigbio/mediqa_qa
+    Paper: https://aclanthology.org/W19-5039/
+
+    Sample Prompt:
+        Answer the following consumer health question.
+
+        Question: Noonan syndrome. What are the references with noonan syndrome
+        and polycystic renal disease?
+        Answer:
+
+    @inproceedings{MEDIQA2019,
+        author    = {Asma {Ben Abacha} and Chaitanya Shivade and Dina Demner{-}Fushman},
+        title     = {Overview of the MEDIQA 2019 Shared Task on Textual Inference,
+                     Question Entailment and Question Answering},
+        booktitle = {ACL-BioNLP 2019},
+        year      = {2019}
+    }
+    """
+
+    name = "medi_qa"
+    description = (
+        "MEDIQA is a benchmark designed to evaluate a model's ability to generate"
+        "medically accurate answers to patient-generated questions. Each instance includes a"
+        "consumer health question, a set of candidate answers (used in ranking tasks), relevance"
+        "annotations, and optionally, additional context. The benchmark focuses on supporting"
+        "patient understanding and accessibility in health communication."
+    )
+    tags = ["knowledge", "biomedical"]
+
+    def _get_highest_ranked_answer(self, answers: List[Dict[str, Dict[str, str]]]) -> str:
+        best_answer: str = ""
+        for answer in answers:
+            if answer["Answer"]["ReferenceRank"] != 1:
+                continue
+            best_answer = answer["Answer"]["AnswerText"]
+            break
+        return best_answer
+
+    def process_csv(self, data, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        hlog(f"Processing data for {split} split")
+        total_tokens: int = 0
+        counter = 0
+        for row in data:
+            row = row["QUESTION"]
+            question = row["QuestionText"]
+            ground_truth_answer = self._get_highest_ranked_answer(row["AnswerList"])
+            id = row["QID"]
+            counter += 1
+            total_tokens += len(ground_truth_answer.split())
+            instances.append(
+                Instance(
+                    input=Input(question),
+                    references=[Reference(Output(ground_truth_answer), tags=[CORRECT_TAG])],
+                    split=split,
+                    id=id,
+                )
+            )
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Load the MEDIQA dataset from Hugging Face
+        dataset = load_dataset(
+            "bigbio/mediqa_qa",
+            trust_remote_code=True,
+            revision="9288641f4c785c95dc9079fa526dabb12efdb041",
+        )
+
+        # Process all the instances
+        instances: List[Instance] = []
+        # Limit to zero shot setting
+        splits: Dict[str, str] = {
+            # "train_live_qa_med": TRAIN_SPLIT,
+            # "validation": VALID_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for hf_split, split in splits.items():
+            data = dataset[hf_split]
+            instances.extend(self.process_csv(data, split))
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medi_qa",
+            display_name="MEDIQA",
+            description="MEDIQA is a benchmark designed to evaluate a model's ability to retrieve and "
+            "generate medically accurate answers to patient-generated questions. Each "
+            "instance includes a consumer health question, a set of candidate answers (used "
+            "in ranking tasks), relevance annotations, and optionally, additional context. "
+            "The benchmark focuses on supporting patient understanding and accessibility in "
+            "health communication.",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate medically accurate answers to patient-generated questions.",
+                when="Any",
+                who="Clinician, Medical Student",
+                language="English",
+            ),
+            main_metric="medi_qa_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/medication_qa_scenario.py

@@ -0,0 +1,96 @@
+import os
+from typing import List
+
+import pandas as pd
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import ensure_file_downloaded
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Instance,
+    Output,
+    Reference,
+    Scenario,
+    ScenarioMetadata,
+)
+
+
+class MedicationQAScenario(Scenario):
+    """
+    The gold standard corpus for medication question answering introduced in the MedInfo 2019 paper
+    "Bridging the Gap between Consumers’ Medication Questions and Trusted Answers":
+    http://ebooks.iospress.nl/publication/51941
+
+    This dataset has consumer questions, as opposed to very clinical questions.
+
+    Paper citation:
+
+        @inproceedings{BenAbacha:MEDINFO19,
+        author    = {Asma {Ben Abacha} and Yassine Mrabet and Mark Sharp and
+                    Travis Goodwin and Sonya E. Shooshan and Dina Demner{-}Fushman},
+        title     = {Bridging the Gap between Consumers’ Medication Questions and Trusted Answers},
+        booktitle = {MEDINFO 2019},
+        year      = {2019},
+        }
+    """
+
+    SOURCE_REPO_URL = "https://github.com/abachaa/Medication_QA_MedInfo2019/raw/master/"
+    FILENAME = "MedInfo2019-QA-Medications.xlsx"
+
+    name = "medication_qa"
+    description = (
+        "MedicationQA is a benchmark composed of open-ended consumer health questions"
+        "specifically focused on medications. Each example consists of a free-form question"
+        "and a corresponding medically grounded answer. The benchmark evaluates a model's"
+        "ability to provide accurate, accessible, and informative medication-related responses"
+        "for a lay audience."
+    )
+    tags = ["knowledge", "generation", "question_answering", "biomedical"]
+
+    def download_medication_qa(self, path: str):
+        """download the .xlsx spreadsheet containing the question-answer pairs"""
+        ensure_file_downloaded(
+            source_url=os.path.join(self.SOURCE_REPO_URL, self.FILENAME),
+            target_path=os.path.join(path, self.FILENAME),
+            unpack=False,
+        )
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        self.download_medication_qa(output_path)
+        data_path = os.path.join(output_path, self.FILENAME)
+
+        data = pd.read_excel(data_path)
+        data = data[~data.Answer.isna()]  # remove rows missing answers
+        instances = [
+            Instance(
+                input=Input(row.Question),
+                references=[Reference(Output(row.Answer), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            for _, row in data.iterrows()
+        ]
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medication_qa",
+            display_name="MedicationQA",
+            description="MedicationQA is a benchmark composed of open-ended consumer health questions "
+            "specifically focused on medications. Each example consists of a free-form "
+            "question and a corresponding medically grounded answer. The benchmark "
+            "evaluates a model's ability to provide accurate, accessible, and informative "
+            "medication-related responses for a lay audience.",
+            taxonomy=TaxonomyInfo(
+                task="Question answering",
+                what="Answer consumer medication-related questions",
+                when="Any",
+                who="Patient, Pharmacist",
+                language="English",
+            ),
+            main_metric="medication_qa_accuracy",
+            main_split="test",
+        )