crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/medbullets_scenario.py

@@ -67,8 +67,13 @@ class MedBulletsScenario(Scenario):
         "https://raw.githubusercontent.com/HanjieChen/ChallengeClinicalQA/refs/heads/main/medbullets/"
     )
 
-    name = "medbullet"
-    description = "A USMLE-style medical question dataset with multiple-choice answers and explanations."
+    name = "medbullets"
+    description = (
+        "Medbullets is a benchmark of USMLE-style medical questions designed to assess a"
+        "model’s ability to understand and apply clinical knowledge. Each question is accompanied"
+        "by a patient scenario and five multiple-choice options, similar to those found on"
+        "Step 2 and Step 3 on the US medical licensing exam."
+    )
     tags = ["reasoning", "biomedical"]
 
     # Define the possible answer choices

helm/benchmark/scenarios/medcalc_bench_scenario.py

@@ -71,8 +71,10 @@ class MedCalcBenchScenario(Scenario):
 
     name = "medcalc_bench"
     description = (
-        "A dataset which consists of a patient note, a question "
-        "requesting to compute a specific medical value, and a ground truth answer."
+        "MedCalc-Bench is a benchmark designed to evaluate models on their ability to compute"
+        "clinically relevant values from patient notes. Each instance consists of a clinical note"
+        "describing the patient's condition, a diagnostic question targeting a specific medical"
+        "value, and a ground truth response."
     )
     tags = ["knowledge", "reasoning", "biomedical"]
 

helm/benchmark/scenarios/medec_scenario.py

@@ -50,7 +50,12 @@ class MedecScenario(Scenario):
     TEST_URL = f"https://raw.githubusercontent.com/abachaa/MEDEC/{GIT_HASH}/MEDEC-MS/MEDEC-MS-TestSet-with-GroundTruth-and-ErrorType.csv"  # noqa: E501
 
     name = "medec"
-    description = "A dataset containing medical narratives with error detection and correction pairs."
+    description = (
+        "Medec is a benchmark composed of clinical narratives that include either correct"
+        "documentation or medical errors. Each entry includes sentence-level identifiers and an"
+        "associated correction task. The model must review the narrative and either identify"
+        "the erroneous sentence and correct it, or confirm that the text is entirely accurate."
+    )
     tags = ["error_detection", "error_correction", "biomedical"]
 
     def download_csv(self, url: str, output_path: str, file_name: str) -> str:

helm/benchmark/scenarios/medhallu_scenario.py

@@ -20,7 +20,13 @@ class MedHalluScenario(Scenario):
     """
 
     name = "medhallu"
-    description = "A dataset of PubMed articles and associated questions, with the objective being to classify whether the answer is factual or hallucinated."  # noqa: E501
+    description = (
+        "MedHallu is a benchmark focused on evaluating factual correctness in biomedical"
+        "question answering. Each instance contains a PubMed-derived knowledge snippet, a"
+        "biomedical question, and a model-generated answer. The task is to classify whether the"
+        "answer is factually correct or contains hallucinated (non-grounded) information. This"
+        "benchmark is designed to assess the factual reliability of medical language models."
+    )
     tags = ["knowledge", "reasoning", "biomedical"]
 
     def create_instance(self, question, knowledge, answer, label, split):

helm/benchmark/scenarios/medi_qa_scenario.py

@@ -49,9 +49,11 @@ class MediQAScenario(Scenario):
 
     name = "medi_qa"
     description = (
-        "A dataset including a medical question, a set of candidate answers,"
-        "relevance annotations for ranking, and additional context to evaluate understanding"
-        "and retrieval capabilities in a healthcare setting."
+        "MEDIQA is a benchmark designed to evaluate a model's ability to retrieve and generate"
+        "medically accurate answers to patient-generated questions. Each instance includes a"
+        "consumer health question, a set of candidate answers (used in ranking tasks), relevance"
+        "annotations, and optionally, additional context. The benchmark focuses on supporting"
+        "patient understanding and accessibility in health communication."
     )
     tags = ["knowledge", "biomedical"]
 
@@ -88,7 +90,11 @@ class MediQAScenario(Scenario):
 
     def get_instances(self, output_path: str) -> List[Instance]:
         # Load the MEDIQA dataset from Hugging Face
-        dataset = load_dataset("bigbio/mediqa_qa")
+        dataset = load_dataset(
+            "bigbio/mediqa_qa",
+            trust_remote_code=True,
+            revision="9288641f4c785c95dc9079fa526dabb12efdb041",
+        )
 
         # Process all the instances
         instances: List[Instance] = []

helm/benchmark/scenarios/medication_qa_scenario.py

@@ -31,7 +31,13 @@ class MedicationQAScenario(Scenario):
     FILENAME = "MedInfo2019-QA-Medications.xlsx"
 
     name = "medication_qa"
-    description = "Open text question-answer pairs regarding consumer health questions about medication."
+    description = (
+        "MedicationQA is a benchmark composed of open-ended consumer health questions"
+        "specifically focused on medications. Each example consists of a free-form question"
+        "and a corresponding medically grounded answer. The benchmark evaluates a model's"
+        "ability to provide accurate, accessible, and informative medication-related responses"
+        "for a lay audience."
+    )
     tags = ["knowledge", "generation", "question_answering", "biomedical"]
 
     def download_medication_qa(self, path: str):

helm/benchmark/scenarios/melt_ir_scenario.py

@@ -0,0 +1,171 @@
+from typing import List, Optional
+
+from datasets import load_dataset, Dataset
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    make_rank_tag,
+)
+
+
+class MELTInformationRetrievalScenario(Scenario):
+    name = "melt_information_retrieval"
+    description = "Scenario for information retrieval tasks."
+    tags = ["information_retrieval"]
+
+    """ Dictionary mapping task track tuples to the number of queries. """
+    NUM_TRAIN_QUERIES = 1000
+
+    """ Upper and lower bounds on top-k.
+
+    The top-k number represents the number of passages we will consider per
+    query. Max top-k for the train and validation files are set to the number
+    of passages included in the corresponding top-k files.
+    """
+    MIN_TOPK: int = 11
+    MAX_TRAIN_TOPK: int = 20
+    MAX_VALID_TOPK: int = 1000
+
+    def __init__(
+        self, dataset_name: str, revision: str, subset: Optional[str] = None, valid_topk: Optional[int] = None
+    ):
+        """The constructor for the MSMARCOScenario.
+
+        Args:
+            dataset_name: The name of the dataset.
+            revision: The revision of the dataset to use.
+            subset: The subset of the dataset to use. Defaults to "".
+            valid_topk: If set, specifies the number of top documents for which the
+                validation instances will be created. Must be in the range
+                [self.MIN_TOPK, self.MAX_VALID_TOPK].
+        """
+        super().__init__()
+
+        # Input validation
+        self.dataset_name = dataset_name
+        self.revision = revision
+        self.subset = subset
+        self.valid_topk: Optional[int] = valid_topk
+        if self.valid_topk is not None:
+            assert valid_topk and self.MIN_TOPK <= valid_topk <= self.MAX_VALID_TOPK
+
+    def get_train_instances(self) -> List[Instance]:
+        """Get training instances.
+        References for each instance are selected as follows:
+            1. We select 1 correct reference, where the documents included
+            corresponds to the best document for the given train query.
+            2. We create 1 wrong reference, where the document included
+            corresponds to a non-gold document for the given train query.
+        """
+        dataset = load_dataset(
+            self.dataset_name,
+            self.subset,
+            revision=self.revision,
+            trust_remote_code=True,
+        )
+        instances = []
+        for i, sample in enumerate(dataset["train"]):
+
+            if i >= self.NUM_TRAIN_QUERIES:
+                break
+
+            references = [
+                Reference(Output(text=sample["positive"]), tags=[CORRECT_TAG]),
+                Reference(Output(text=sample["negative"]), tags=[]),
+            ]
+
+            instances.append(Instance(Input(text=sample["query"]), references=references, split=TRAIN_SPLIT))
+        return instances
+
+    def get_valid_instances(self) -> List[Instance]:
+        """Get validation instances.
+        By default, we create a reference for each Document ID for which there
+        is a judgment with respect to the provided Query ID.
+
+        If self.valid_topk is not None, we ensure that a reference is created
+        for all the documents that appear in top self.valid_topk documents for
+        the given validation query.
+        """
+        dataset = load_dataset(
+            self.dataset_name,
+            f"runs-{self.subset}",
+            revision=self.revision,
+            trust_remote_code=True,
+        )
+        instances = []
+        for sample in dataset["bm25"]:
+            references = []
+
+            for k, passage_dict in enumerate(Dataset.from_dict(sample["passages"])):
+                if self.valid_topk is None or k >= self.valid_topk:
+                    break
+                tags = []
+                tags.append(f"docid={passage_dict['id']}")
+                if k == 0:
+                    tags.append(CORRECT_TAG)
+                tags.append(make_rank_tag(rank=k + 1))  # Top-k rank
+                references.append(Reference(Output(text=passage_dict["passage"]), tags=tags))
+
+            instances.append(Instance(Input(text=sample["query"]), references=references, split=VALID_SPLIT))
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Get instances for this scenario.
+
+        Refer to the documentation of the following methods for details on how
+        the instances are created:
+            * self.get_train_instances
+            * self.get_valid_instances
+        """
+
+        hlog("Preparing training instances.")
+        train_instances = self.get_train_instances()
+
+        hlog("Preparing validation instances.")
+        valid_instances = self.get_valid_instances()
+
+        return train_instances + valid_instances
+
+
+class MELTInformationRetrievalMMARCOScenario(MELTInformationRetrievalScenario):
+    """
+    Scenario for the MMARCO dataset.
+    """
+
+    name = "melt_information_retrieval_mmarco"
+    description = "MMARCO dataset for information retrieval in Vietnamese."
+    tags = ["information_retrieval"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="unicamp-dl/mmarco",
+            revision="6d039c4638c0ba3e46a9cb7b498b145e7edc6230",
+            subset="vietnamese",
+            **kwargs,
+        )
+
+
+class MELTInformationRetrievalMRobustScenario(MELTInformationRetrievalScenario):
+    """
+    Scenario for the MRobust dataset.
+    """
+
+    name = "melt_information_retrieval_mrobust"
+    description = "MRobust dataset for information retrieval in Vietnamese."
+    tags = ["information_retrieval"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="unicamp-dl/mrobust",
+            revision="fda452a7fbfd9550db2f78d9d98e6b3ec16734df",
+            subset="vietnamese",
+            **kwargs,
+        )

helm/benchmark/scenarios/melt_knowledge_scenario.py

@@ -0,0 +1,246 @@
+from abc import abstractmethod
+from typing import Dict, List, Tuple, Optional
+
+import random
+from datasets import load_dataset
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    PassageQuestionInput,
+    Input,
+    Output,
+)
+
+
+class MELTClosedBookQAScenario(Scenario):
+    name = "melt_closed_book_qa"
+    description = "Closed Book Question Answering scenario."
+    tags = ["question_answering"]
+
+    def __init__(
+        self,
+        dataset_name: str,
+        revision: str,
+        subset: Optional[str] = None,
+        splits: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Initializes the question answering scenario.
+
+        Args:
+            dataset_name: The name of the dataset.
+            revision: The revision of the dataset to use.
+            subset: The subset of the dataset to use. Defaults to "".
+            splits: The splits to use for the dataset. Defaults to None.
+        """
+        super().__init__()
+        self.dataset_name = dataset_name
+        self.subset = subset
+        self.revision = revision
+        self.splits = splits
+
+    def get_instances_for_splits(self, splits: Dict[str, str]) -> List[Instance]:
+        """
+        Helper for generating instances for a split.
+        Args:
+            splits (dict): Which splits to partition the data into.
+        Returns:
+            List[Instance]: Instances from the file for the specified split.
+        """
+        instances: List[Instance] = []
+        dataset = load_dataset(
+            self.dataset_name,
+            self.subset,
+            revision=self.revision,
+            trust_remote_code=True,
+        )
+        for dataset_split_name, helm_split_name in splits.items():
+
+            for sample in dataset[dataset_split_name]:
+                instance = Instance(
+                    input=Input(text=sample["question"]),
+                    references=[Reference(Output(text=sample["answer"]), tags=[CORRECT_TAG])],
+                    split=helm_split_name,
+                )
+                instances.append(instance)
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        if self.splits is None:
+            splits = {"train": TRAIN_SPLIT, "validation": VALID_SPLIT, "test": TEST_SPLIT}
+        else:
+            splits = {}
+            if "train" in self.splits:
+                splits[self.splits[TRAIN_SPLIT]] = TRAIN_SPLIT
+            if "validation" in self.splits:
+                splits[self.splits[VALID_SPLIT]] = VALID_SPLIT
+            if "test" in self.splits:
+                splits[self.splits[TEST_SPLIT]] = TEST_SPLIT
+
+        instances: List[Instance] = self.get_instances_for_splits(splits=splits)
+        return instances
+
+
+class MELTKnowledgeZaloScenario(MELTClosedBookQAScenario):
+    """
+    Scenario for the Zalo dataset.
+    """
+
+    name = "melt_knowledge_zalo"
+    description = "Zalo dataset for closed-book question answering."
+    tags = ["question_answering", "knowledge"]
+
+    def __init__(self):
+        super().__init__(
+            dataset_name="ura-hcmut/zalo_e2eqa",
+            revision="63494521f4de949bfa57a5f0b79bc3ee47e635ad",
+            splits={
+                TRAIN_SPLIT: "train",
+                TEST_SPLIT: "test",
+            },
+        )
+
+
+class MELTMultipleChoiceQAScenario(Scenario):
+    name = "melt_multiple_choice_qa"
+    description = "Multiple Choice Question Answering scenario."
+    tags = ["question_answering"]
+
+    def __init__(
+        self,
+        dataset_name: str,
+        revision: str,
+        subset: Optional[str] = None,
+        splits: Optional[Dict[str, str]] = None,
+    ):
+        """
+        Initializes the question answering scenario.
+
+        Args:
+            dataset_name: The name of the dataset.
+            revision: The revision of the dataset to use.
+            subset: The subset of the dataset to use. Defaults to "".
+            splits: The splits to use for the dataset. Defaults to None.
+        """
+        super().__init__()
+        self.dataset_name = dataset_name
+        self.subset = subset
+        self.revision = revision
+        self.splits = splits
+
+    @abstractmethod
+    def process_example(self, sample: dict) -> Tuple[Input, List[Reference]]:
+        """
+        Given an sample from the dataset, create the input text and
+        list of answers for the instance.
+        """
+        pass
+
+    def get_instances_for_splits(self, splits: Dict[str, str]) -> List[Instance]:
+        """
+        Helper for generating instances for a split.
+        Args:
+            splits (dict): Which splits to partition the data into.
+        Returns:
+            List[Instance]: Instances from the file for the specified split.
+        """
+        instances: List[Instance] = []
+        dataset = load_dataset(
+            self.dataset_name,
+            self.subset,
+            revision=self.revision,
+            trust_remote_code=True,
+        )
+        for dataset_split_name, helm_split_name in splits.items():
+            for sample in dataset[dataset_split_name]:
+                inputs, references = self.process_example(sample)
+                instance = Instance(
+                    input=inputs,
+                    references=references,
+                    split=helm_split_name,
+                )
+                instances.append(instance)
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        if self.splits is None:
+            splits = {"train": TRAIN_SPLIT, "validation": VALID_SPLIT, "test": TEST_SPLIT}
+        else:
+            splits = {}
+            if "train" in self.splits:
+                splits[self.splits[TRAIN_SPLIT]] = TRAIN_SPLIT
+            if "validation" in self.splits:
+                splits[self.splits[VALID_SPLIT]] = VALID_SPLIT
+            if "test" in self.splits:
+                splits[self.splits[TEST_SPLIT]] = TEST_SPLIT
+
+        instances: List[Instance] = self.get_instances_for_splits(splits=splits)
+        return instances
+
+
+class MELTKnowledgeViMMRCScenario(MELTMultipleChoiceQAScenario):
+    """
+    Scenario for the ViMMRC dataset.
+    """
+
+    name = "melt_knowledge_vimmrc"
+    description = "ViMMRC dataset for multiple choice question answering."
+    tags = ["question_answering", "knowledge"]
+
+    def __init__(self, randomize_order: bool = False):
+        super().__init__(
+            dataset_name="ura-hcmut/ViMMRC",
+            revision="fe68800e37aaa84d80b1d93466b36c3fa60d8bcb",
+            splits={
+                TRAIN_SPLIT: "train",
+                VALID_SPLIT: "validation",
+                TEST_SPLIT: "test",
+            },
+        )
+        self.randomize_order = randomize_order
+        self.correct_answer_mapping = {
+            "A": 0,
+            "B": 1,
+            "C": 2,
+            "D": 3,
+            "E": 4,
+            "F": 5,
+            "G": 6,
+            "H": 7,
+        }
+        random.seed(42)
+
+    def process_example(self, sample: dict) -> Tuple[Input, List[Reference]]:
+        """
+        Given an sample from the dataset, create the input text and
+        list of answers for the instance.
+        """
+        inputs = PassageQuestionInput(
+            passage=sample["article"],
+            passage_prefix="Ngữ cảnh: ",
+            question=sample["question"],
+            question_prefix="Câu hỏi: ",
+            separator="\n\n",
+        )
+
+        correct_idx = self.correct_answer_mapping[sample["answer"]]
+        references = []
+        for idx, answer in enumerate(eval(sample["options"])):
+            if idx == correct_idx:
+                tags = [CORRECT_TAG]
+            else:
+                tags = []
+
+            references.append(Reference(Output(text=answer), tags=tags))
+
+        if self.randomize_order:
+            random.shuffle(references)
+
+        return inputs, references