crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/scenarios/medcalc_bench_scenario.py

@@ -0,0 +1,127 @@
+from typing import Dict, List
+from datasets import load_dataset
+
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    PassageQuestionInput,
+    Output,
+)
+
+
+class MedCalcBenchScenario(Scenario):
+    """
+    MedCalc-Bench is the first medical calculation dataset used to benchmark
+    LLMs ability to serve as clinical calculators.
+    Each instance in the dataset consists of a patient note, a question asking
+    to compute a specific clinical value, a final answer value, and a step-by-step
+    solution explaining how the final answer was obtained. Our dataset covers 55
+    different calculation tasks. We hope this dataset serves as a call to improve
+    the verbal and computational reasoning skills of LLMs in medical settings.
+
+    This dataset contains a training dataset of 10,053 instances and a testing
+    dataset of 1,047 instances.
+
+    Dataset: https://huggingface.co/datasets/ncbi/MedCalc-Bench-v1.0
+    Paper: https://arxiv.org/abs/2406.12036
+
+    Sample Prompt:
+        Given a patient note and a clinical question, compute the requested medical value.
+        Be as concise as possible.
+
+        Patient note: A 70-year-old female was rushed into the ICU due to respiratory distress,
+        following which she was promptly put on mechanical ventilation. Her delivered oxygen fell
+        to 51 % FiO₂; meanwhile, her partial pressure of oxygen (PaO₂) registered at 74 mm Hg.
+        She was conscious but visibly disoriented with a functional Glasgow Coma Score of 12.
+        She was hypotensive with blood pressure of 91/70 mm Hg. Multiple vasopressors are being administered
+        simultaneously including DOPamine at 4 mcg/kg/min, norEPINEPHrine at 0.06 mcg/kg/min,
+        DOBUTamine at 3 mcg/kg/min, and EPINEPHrine at 0.03 mcg/kg/min. Laboratory evaluations
+        revealed mild renal impairment with creatinine levels slightly elevated at 1.6 mg/dL
+        and a bilirubin level of 1.9 mg/dL. Her platelet count was found to be 165,000/µL.
+        Her daily urine output of 950 mL.
+        Question: What is the patient's Sequential Organ Failure Assessment (SOFA) Score?
+
+        Answer:
+
+    @misc{khandekar2024medcalcbench,
+        title={MedCalc-Bench: Evaluating Large Language Models for Medical Calculations},
+        author={
+            Nikhil Khandekar and Qiao Jin and Guangzhi Xiong and Soren Dunn and Serina S Applebaum and
+            Zain Anwar and Maame Sarfo-Gyamfi and Conrad W Safranek and Abid A Anwar and Andrew Zhang and
+            Aidan Gilson and Maxwell B Singer and Amisha Dave and Andrew Taylor and Aidong Zhang and
+            Qingyu Chen and Zhiyong Lu
+        },
+        year={2024},
+        eprint={2406.12036},
+        archivePrefix={arXiv},
+        primaryClass={
+            id='cs.CL' full_name='Computation and Language' is_active=True alt_name='cmp-lg'
+            in_archive='cs' is_general=False description='Covers natural language processing.
+            Roughly includes material in ACM Subject Class I.2.7. Note that work on artificial
+            languages (programming languages, logics, formal systems) that does not explicitly
+            address natural-language issues broadly construed (natural-language processing, computational
+            linguistics, speech, text retrieval, etc.) is not appropriate for this area.'
+        }
+    }
+    """
+
+    name = "medcalc_bench"
+    description = (
+        "MedCalc-Bench is a benchmark designed to evaluate models on their ability to compute"
+        "clinically relevant values from patient notes. Each instance consists of a clinical note"
+        "describing the patient's condition, a diagnostic question targeting a specific medical"
+        "value, and a ground truth response."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    def __init__(self):
+        super().__init__()
+
+    def process_csv(self, data, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        hlog(f"Processing data for {split} split")
+        for row in data:
+            question = row["Question"]
+            ground_truth_answer = row["Ground Truth Answer"]
+            patient_note = row["Patient Note"]
+            id = row["Row Number"]
+
+            prompt = PassageQuestionInput(
+                passage=patient_note + "\n", question=question + "\n", passage_prefix="Patient note: "
+            )
+
+            extra_data = {
+                "category": row["Category"],
+                "upper_limit": row["Upper Limit"],
+                "lower_limit": row["Lower Limit"],
+            }
+
+            instance = Instance(
+                input=prompt,
+                references=[Reference(Output(text=ground_truth_answer), tags=[CORRECT_TAG])],
+                extra_data=extra_data,
+                split=split,
+                id=id,
+            )
+            instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Load the MedCalc-Bench dataset from Hugging Face
+        dataset = load_dataset("ncbi/MedCalc-Bench-v1.0")
+
+        # Process all the instances - limit to zero shot setting
+        instances: List[Instance] = []
+        splits: Dict[str, str] = {
+            # "train": TRAIN_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for hf_split, split in splits.items():
+            data = dataset[hf_split]
+            instances.extend(self.process_csv(data, split))
+
+        return instances

helm/benchmark/scenarios/medec_scenario.py

@@ -0,0 +1,125 @@
+import csv
+import os
+from typing import List
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_file_downloaded
+
+
+class MedecScenario(Scenario):
+    """
+    Processes the MEDEC dataset for medical error detection and correction tasks.
+
+    MEDEC is the first publicly available benchmark for medical error detection and correction in clinical notes,
+    introduced in "Ben Abacha et al., 2024." The dataset includes 3,848 clinical texts from the MS and UW collections,
+    covering five types of errors:
+    - Diagnosis
+    - Management
+    - Treatment
+    - Pharmacotherapy
+    - Causal Organism
+
+    The dataset consists of:
+    - Training Set: 2,189 MS texts
+    - Validation Set: 574 MS texts and 160 UW texts
+    - Test Set: 597 MS texts and 328 UW texts
+
+    Each clinical text is labeled as either correct or containing one error. The task involves:
+    (A) Predicting the error flag (1: the text contains an error, 0: the text has no errors).
+    (B) For flagged texts, extracting the sentence that contains the error.
+    (C) Generating a corrected sentence.
+
+    The MEDEC dataset was used for the MEDIQA-CORR shared task to evaluate seventeen participating systems.
+    Recent LLMs (e.g., GPT-4, Claude 3.5 Sonnet, Gemini 2.0 Flash) have been evaluated on this dataset, showing good
+    performance but still lagging behind medical doctors in error detection and correction tasks.
+
+    Task:
+    Given a clinical text, models must identify errors and correct them while demonstrating medical knowledge
+    and reasoning capabilities.
+    """
+
+    GIT_HASH = "49c59dcba43a7af8717590a405b11a57f42b04df"
+    TRAIN_URL = f"https://raw.githubusercontent.com/abachaa/MEDEC/{GIT_HASH}/MEDEC-MS/MEDEC-Full-TrainingSet-with-ErrorType.csv"  # noqa: E501
+    TEST_URL = f"https://raw.githubusercontent.com/abachaa/MEDEC/{GIT_HASH}/MEDEC-MS/MEDEC-MS-TestSet-with-GroundTruth-and-ErrorType.csv"  # noqa: E501
+
+    name = "medec"
+    description = (
+        "Medec is a benchmark composed of clinical narratives that include either correct"
+        "documentation or medical errors. Each entry includes sentence-level identifiers and an"
+        "associated correction task. The model must review the narrative and either identify"
+        "the erroneous sentence and correct it, or confirm that the text is entirely accurate."
+    )
+    tags = ["error_detection", "error_correction", "biomedical"]
+
+    def download_csv(self, url: str, output_path: str, file_name: str) -> str:
+        """Download the CSV file and return its path."""
+        csv_path = os.path.join(output_path, file_name)
+        ensure_file_downloaded(source_url=url, target_path=csv_path, unpack=False)
+        return csv_path
+
+    def process_csv(self, csv_path: str, split: str) -> List[Instance]:
+        """Read and process a CSV file to generate instances."""
+        instances: List[Instance] = []
+        with open(csv_path, "r", encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+
+            for row in reader:
+                # Ensure required fields exist
+                if not row.get("Sentences"):
+                    continue
+
+                # Extract text content
+                input_text = row["Sentences"].strip()
+
+                # Extract metadata
+                error_flag = int(row.get("Error Flag", 0))
+                error_sentence_id = row.get("Error Sentence ID", "-1").strip()
+                corrected_sentence = row.get("Corrected Sentence", "").strip()
+
+                references = []
+                if error_flag == 1 and corrected_sentence != "NA" and error_sentence_id != "-1":
+                    # If there is an error, store the corrected sentence
+                    references.append(
+                        Reference(
+                            Output(text=f"{error_sentence_id} {corrected_sentence}"),
+                            tags=[CORRECT_TAG],
+                        )
+                    )
+                else:
+                    # If no error exists, expect "CORRECT" as the output
+                    references.append(
+                        Reference(
+                            Output(text="CORRECT"),
+                            tags=[CORRECT_TAG],
+                        )
+                    )
+
+                instance = Instance(
+                    input=Input(text=input_text),
+                    references=references,
+                    split=split,
+                )
+                instances.append(instance)
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Download and process the dataset to generate instances."""
+        instances: List[Instance] = []
+
+        # Download and process the training set - limit to zero shot setting for now
+        # train_csv = self.download_csv(self.TRAIN_URL, output_path, "medec_train.csv")
+        # instances.extend(self.process_csv(train_csv, TRAIN_SPLIT))
+
+        # Download and process the test set
+        test_csv = self.download_csv(self.TEST_URL, output_path, "medec_test.csv")
+        instances.extend(self.process_csv(test_csv, TEST_SPLIT))
+
+        return instances

helm/benchmark/scenarios/medhallu_scenario.py

@@ -0,0 +1,72 @@
+from typing import List
+from datasets import load_dataset
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Output,
+    Input,
+)
+
+
+class MedHalluScenario(Scenario):
+    """
+    MedHallu is a medical hallucination dataset that consists of PubMed articles and associated questions,
+    with the objective being to classify whether the answer is factual or hallucinated.
+    MedHallu: https://medhallu.github.io/
+    """
+
+    name = "medhallu"
+    description = (
+        "MedHallu is a benchmark focused on evaluating factual correctness in biomedical"
+        "question answering. Each instance contains a PubMed-derived knowledge snippet, a"
+        "biomedical question, and a model-generated answer. The task is to classify whether the"
+        "answer is factually correct or contains hallucinated (non-grounded) information. This"
+        "benchmark is designed to assess the factual reliability of medical language models."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    def create_instance(self, question, knowledge, answer, label, split):
+        prompt_text = f"""World Knowledge: {knowledge}
+
+Question: {question}
+
+Answer: {answer}
+"""
+        return Instance(
+            input=Input(
+                text=prompt_text,
+            ),
+            references=[Reference(Output(text=label), tags=[CORRECT_TAG])],
+            split=split,
+        )
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Load the MedCalc-Bench dataset from Hugging Face
+        dataset = load_dataset(
+            "UTAustin-AIHealth/MedHallu",
+            "pqa_labeled",
+            split="train",
+            revision="515060458a945c633debc6fd5baac7764416b724",
+        )
+
+        # Process all the instances - limit to zero shot setting
+        instances: List[Instance] = []
+        for row in dataset:
+            # print("printing row", row)
+            question = row["Question"]
+            ground_truth_answer = row["Ground Truth"]
+            knowledge = row["Knowledge"]
+            hallucinated_answer = row["Hallucinated Answer"]
+
+            gt_instance = self.create_instance(question, knowledge, ground_truth_answer, label="0", split=TEST_SPLIT)
+            instances.append(gt_instance)
+
+            hallucinated_instance = self.create_instance(
+                question, knowledge, hallucinated_answer, label="1", split=TEST_SPLIT
+            )
+            instances.append(hallucinated_instance)
+        return instances

helm/benchmark/scenarios/medi_qa_scenario.py

@@ -0,0 +1,111 @@
+from typing import Dict, List
+from datasets import load_dataset
+
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MediQAScenario(Scenario):
+    """
+    MEDIQA-QA is a dataset designed to benchmark large language models (LLMs) on medical
+    question answering (QA) tasks.
+    Each instance in the dataset includes a medical question, a set of candidate answers,
+    relevance annotations for ranking, and additional context to evaluate understanding
+    and retrieval capabilities in a healthcare setting.
+
+    The dataset encompasses diverse question types, including consumer health queries
+    and clinical questions, making it suitable for assessing LLMs' ability to answer
+    consumer healthcare questions.
+
+    This dataset comprises two training sets of 104 instances each, a validation set
+    of 25 instances, and a testing set of 150 instances.
+
+    Dataset: https://huggingface.co/datasets/bigbio/mediqa_qa
+    Paper: https://aclanthology.org/W19-5039/
+
+    Sample Prompt:
+        Answer the following consumer health question.
+
+        Question: Noonan syndrome. What are the references with noonan syndrome
+        and polycystic renal disease?
+        Answer:
+
+    @inproceedings{MEDIQA2019,
+        author    = {Asma {Ben Abacha} and Chaitanya Shivade and Dina Demner{-}Fushman},
+        title     = {Overview of the MEDIQA 2019 Shared Task on Textual Inference,
+                     Question Entailment and Question Answering},
+        booktitle = {ACL-BioNLP 2019},
+        year      = {2019}
+    }
+    """
+
+    name = "medi_qa"
+    description = (
+        "MEDIQA is a benchmark designed to evaluate a model's ability to retrieve and generate"
+        "medically accurate answers to patient-generated questions. Each instance includes a"
+        "consumer health question, a set of candidate answers (used in ranking tasks), relevance"
+        "annotations, and optionally, additional context. The benchmark focuses on supporting"
+        "patient understanding and accessibility in health communication."
+    )
+    tags = ["knowledge", "biomedical"]
+
+    def _get_highest_ranked_answer(self, answers: List[Dict[str, Dict[str, str]]]) -> str:
+        best_answer: str = ""
+        for answer in answers:
+            if answer["Answer"]["ReferenceRank"] != 1:
+                continue
+            best_answer = answer["Answer"]["AnswerText"]
+            break
+        return best_answer
+
+    def process_csv(self, data, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        hlog(f"Processing data for {split} split")
+        total_tokens: int = 0
+        counter = 0
+        for row in data:
+            row = row["QUESTION"]
+            question = row["QuestionText"]
+            ground_truth_answer = self._get_highest_ranked_answer(row["AnswerList"])
+            id = row["QID"]
+            counter += 1
+            total_tokens += len(ground_truth_answer.split())
+            instances.append(
+                Instance(
+                    input=Input(question),
+                    references=[Reference(Output(ground_truth_answer), tags=[CORRECT_TAG])],
+                    split=split,
+                    id=id,
+                )
+            )
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Load the MEDIQA dataset from Hugging Face
+        dataset = load_dataset(
+            "bigbio/mediqa_qa",
+            trust_remote_code=True,
+            revision="9288641f4c785c95dc9079fa526dabb12efdb041",
+        )
+
+        # Process all the instances
+        instances: List[Instance] = []
+        # Limit to zero shot setting
+        splits: Dict[str, str] = {
+            # "train_live_qa_med": TRAIN_SPLIT,
+            # "validation": VALID_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for hf_split, split in splits.items():
+            data = dataset[hf_split]
+            instances.extend(self.process_csv(data, split))
+
+        return instances

helm/benchmark/scenarios/medication_qa_scenario.py

@@ -5,7 +5,7 @@ import pandas as pd
 
 from helm.common.general import ensure_file_downloaded
 
-from .scenario import CORRECT_TAG, TEST_SPLIT, Input, Instance, Output, Reference, Scenario
+from helm.benchmark.scenarios.scenario import CORRECT_TAG, TEST_SPLIT, Input, Instance, Output, Reference, Scenario
 
 
 class MedicationQAScenario(Scenario):
@@ -31,7 +31,13 @@ class MedicationQAScenario(Scenario):
     FILENAME = "MedInfo2019-QA-Medications.xlsx"
 
     name = "medication_qa"
-    description = "MedInfo 2019 MedicationQA medication question answering task"
+    description = (
+        "MedicationQA is a benchmark composed of open-ended consumer health questions"
+        "specifically focused on medications. Each example consists of a free-form question"
+        "and a corresponding medically grounded answer. The benchmark evaluates a model's"
+        "ability to provide accurate, accessible, and informative medication-related responses"
+        "for a lay audience."
+    )
     tags = ["knowledge", "generation", "question_answering", "biomedical"]
 
     def download_medication_qa(self, path: str):

helm/benchmark/scenarios/melt_ir_scenario.py

@@ -0,0 +1,171 @@
+from typing import List, Optional
+
+from datasets import load_dataset, Dataset
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    make_rank_tag,
+)
+
+
+class MELTInformationRetrievalScenario(Scenario):
+    name = "melt_information_retrieval"
+    description = "Scenario for information retrieval tasks."
+    tags = ["information_retrieval"]
+
+    """ Dictionary mapping task track tuples to the number of queries. """
+    NUM_TRAIN_QUERIES = 1000
+
+    """ Upper and lower bounds on top-k.
+
+    The top-k number represents the number of passages we will consider per
+    query. Max top-k for the train and validation files are set to the number
+    of passages included in the corresponding top-k files.
+    """
+    MIN_TOPK: int = 11
+    MAX_TRAIN_TOPK: int = 20
+    MAX_VALID_TOPK: int = 1000
+
+    def __init__(
+        self, dataset_name: str, revision: str, subset: Optional[str] = None, valid_topk: Optional[int] = None
+    ):
+        """The constructor for the MSMARCOScenario.
+
+        Args:
+            dataset_name: The name of the dataset.
+            revision: The revision of the dataset to use.
+            subset: The subset of the dataset to use. Defaults to "".
+            valid_topk: If set, specifies the number of top documents for which the
+                validation instances will be created. Must be in the range
+                [self.MIN_TOPK, self.MAX_VALID_TOPK].
+        """
+        super().__init__()
+
+        # Input validation
+        self.dataset_name = dataset_name
+        self.revision = revision
+        self.subset = subset
+        self.valid_topk: Optional[int] = valid_topk
+        if self.valid_topk is not None:
+            assert valid_topk and self.MIN_TOPK <= valid_topk <= self.MAX_VALID_TOPK
+
+    def get_train_instances(self) -> List[Instance]:
+        """Get training instances.
+        References for each instance are selected as follows:
+            1. We select 1 correct reference, where the documents included
+            corresponds to the best document for the given train query.
+            2. We create 1 wrong reference, where the document included
+            corresponds to a non-gold document for the given train query.
+        """
+        dataset = load_dataset(
+            self.dataset_name,
+            self.subset,
+            revision=self.revision,
+            trust_remote_code=True,
+        )
+        instances = []
+        for i, sample in enumerate(dataset["train"]):
+
+            if i >= self.NUM_TRAIN_QUERIES:
+                break
+
+            references = [
+                Reference(Output(text=sample["positive"]), tags=[CORRECT_TAG]),
+                Reference(Output(text=sample["negative"]), tags=[]),
+            ]
+
+            instances.append(Instance(Input(text=sample["query"]), references=references, split=TRAIN_SPLIT))
+        return instances
+
+    def get_valid_instances(self) -> List[Instance]:
+        """Get validation instances.
+        By default, we create a reference for each Document ID for which there
+        is a judgment with respect to the provided Query ID.
+
+        If self.valid_topk is not None, we ensure that a reference is created
+        for all the documents that appear in top self.valid_topk documents for
+        the given validation query.
+        """
+        dataset = load_dataset(
+            self.dataset_name,
+            f"runs-{self.subset}",
+            revision=self.revision,
+            trust_remote_code=True,
+        )
+        instances = []
+        for sample in dataset["bm25"]:
+            references = []
+
+            for k, passage_dict in enumerate(Dataset.from_dict(sample["passages"])):
+                if self.valid_topk is None or k >= self.valid_topk:
+                    break
+                tags = []
+                tags.append(f"docid={passage_dict['id']}")
+                if k == 0:
+                    tags.append(CORRECT_TAG)
+                tags.append(make_rank_tag(rank=k + 1))  # Top-k rank
+                references.append(Reference(Output(text=passage_dict["passage"]), tags=tags))
+
+            instances.append(Instance(Input(text=sample["query"]), references=references, split=VALID_SPLIT))
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Get instances for this scenario.
+
+        Refer to the documentation of the following methods for details on how
+        the instances are created:
+            * self.get_train_instances
+            * self.get_valid_instances
+        """
+
+        hlog("Preparing training instances.")
+        train_instances = self.get_train_instances()
+
+        hlog("Preparing validation instances.")
+        valid_instances = self.get_valid_instances()
+
+        return train_instances + valid_instances
+
+
+class MELTInformationRetrievalMMARCOScenario(MELTInformationRetrievalScenario):
+    """
+    Scenario for the MMARCO dataset.
+    """
+
+    name = "melt_information_retrieval_mmarco"
+    description = "MMARCO dataset for information retrieval in Vietnamese."
+    tags = ["information_retrieval"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="unicamp-dl/mmarco",
+            revision="6d039c4638c0ba3e46a9cb7b498b145e7edc6230",
+            subset="vietnamese",
+            **kwargs,
+        )
+
+
+class MELTInformationRetrievalMRobustScenario(MELTInformationRetrievalScenario):
+    """
+    Scenario for the MRobust dataset.
+    """
+
+    name = "melt_information_retrieval_mrobust"
+    description = "MRobust dataset for information retrieval in Vietnamese."
+    tags = ["information_retrieval"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="unicamp-dl/mrobust",
+            revision="fda452a7fbfd9550db2f78d9d98e6b3ec16734df",
+            subset="vietnamese",
+            **kwargs,
+        )