crfm-helm 0.5.5__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/scenarios/{infinite_bench_sum_scenario.py → infinite_bench_en_sum_scenario.py}

@@ -14,21 +14,19 @@ from helm.benchmark.scenarios.scenario import (
 from helm.common.general import ensure_directory_exists
 
 
-class InfiniteBenchSumScenario(Scenario):
-    """InfiniteBench Sum
+class InfiniteBenchEnSumScenario(Scenario):
+    """InfiniteBench En.Sum
 
     InfiniteBench is a benchmark tailored for evaluating the capabilities of language models to process,
-    understand, and reason over super long contexts (100k+ tokens). InfiniteBench Sum is a subset of
-    InfiniteBench that requires models to generate a concise summary of the novel. The subset is referred
-    to as "En.Sum" in the original paper.
+    understand, and reason over super long contexts (100k+ tokens). InfiniteBench En.Sum is a subset of
+    InfiniteBench that requires models to generate a concise summary of the novel.
     """
 
-    name = "infinite_bench_sum"
-    description = "Summarize a novel from InfiniteBench"
+    name = "infinite_bench_en_sum"
+    description = "∞Bench En.Sum is a summarization task that requires generating a concise summary of a novel. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))"  # noqa: E501
     tags = ["summarization"]
 
-    def __init__(self, min_num_words: int, max_num_words: int):
-        self.min_num_words = min_num_words
+    def __init__(self, max_num_words: int):
         self.max_num_words = max_num_words
         super().__init__()
 
@@ -61,9 +59,9 @@ class InfiniteBenchSumScenario(Scenario):
         def count_words(text: str) -> int:
             return len(re.split(r"\s+", text.strip()))
 
-        dataset = dataset.map(
-            lambda example: {"prompt_wc": count_words(example["context"]) + count_words(example["input"])}
-        ).filter(lambda example: self.min_num_words <= example["prompt_wc"] <= self.max_num_words)
+        dataset = dataset.filter(
+            lambda example: count_words(example["context"]) + count_words(example["input"]) <= self.max_num_words
+        )
 
         # Read all instances
         instances: List[Instance] = []
@@ -75,7 +73,6 @@ class InfiniteBenchSumScenario(Scenario):
                 input=input,
                 references=[Reference(Output(text=row["answer"][0]), tags=[CORRECT_TAG])],
                 split=TEST_SPLIT,
-                extra_data={"word_count": row["prompt_wc"]},
             )
             instances.append(instance)
 

helm/benchmark/scenarios/kpi_edgar_scenario.py

@@ -0,0 +1,151 @@
+import os
+from typing import List, Dict
+import json
+import re
+
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class KPIEDGARScenario(Scenario):
+    """A financial named entity recognition (NER) scenario based on KPI-EDGAR (T. Deußer et al., 2022).
+
+    This scenario has been modified from the paper. The original paper has 12 entity types and requires the model
+    to extract pairs of related entities. This scenario only use four named entity types (kpi, cy, py, py1) and only
+    requires the model to extract individual entities.
+
+    Paper:
+    T. Deußer et al.,
+    “KPI-EDGAR: A Novel Dataset and Accompanying Metric for Relation Extraction from Financial Documents.” 2022.
+    https://arxiv.org/abs/2210.09163
+
+    Prompt format:
+
+    ```
+    Context: {Sentence}
+    Task: Extract key performance indicators (KPIs) and values from the above text. Also, specify one of the following categories to each of the extracted KPIs and values in brackets.
+    kpi: Key Performance Indicators expressible in numerical and monetary value, cy: Current Year monetary value, py: Prior Year monetary value, py1: Two Year Past Value.
+    Answer:
+    ```
+
+    Example input:
+
+    ```
+    Context: The following table summarizes our total share-based compensation expense and excess tax benefits recognized : As of December 28 , 2019 , there was $ 284 million of total unrecognized compensation cost related to nonvested share-based compensation grants .
+    Task: Extract key performance indicators (KPIs) and values from the above text. Also, specify one of the following categories to each of the extracted KPIs and values in brackets.
+    kpi: Key Performance Indicators expressible in numerical and monetary value, cy: Current Year monetary value, py: Prior Year monetary value, py1: Two Year Past Value.
+    Answer:
+    ```
+
+    Example reference:
+    ```
+    284 [cy], total unrecognized compensation cost [kpi]
+    ```"""  # noqa: E501
+
+    name = "kpi_edgar"
+    description = "Named Entity Recognition from financial documents."
+    tags = ["named_entity_recognition", "finance"]
+
+    TAG_DICT = {
+        "kpi": "Key Performance Indicators expressible in numerical and monetary value",
+        "cy": "Current Year monetary value",
+        "py": "Prior Year monetary value",
+        "py1": "Two Year Past Value",
+    }
+    TAG_DESCRIPTIONS = ", ".join(["%s: %s" % (key, val) for (key, val) in TAG_DICT.items()]) + "."
+    TAG_PAREN_RE = (r"\[", r"\]")
+    TAG_PAREN = tuple((e.strip("\\") for e in TAG_PAREN_RE))
+    TAG_PAREN_ESC = ("(", ")")
+    DATASET_SPLIT_TO_HELM_SPLIT = {"train": TRAIN_SPLIT, "valid": VALID_SPLIT, "test": TEST_SPLIT}
+    JSON_URL = "https://raw.githubusercontent.com/tobideusser/kpi-edgar/2ec7084dcd55b4979bbe288d4aa1e962c685c9ab/data/kpi_edgar.json"  # noqa: E501
+    JSON_FILENAME = "kpi_edgar.json"
+
+    @staticmethod
+    def get_sentences(dataset: List[Dict]) -> List[Dict]:
+        return [
+            sentence
+            for document in dataset
+            for segment in document["segments"]
+            for sentence in segment["sentences"] or []
+        ]
+
+    @staticmethod
+    def escape_parenthesis(text: str) -> str:
+        tmp0 = re.sub(KPIEDGARScenario.TAG_PAREN_RE[0], KPIEDGARScenario.TAG_PAREN_ESC[0], text)
+        tmp1 = re.sub(KPIEDGARScenario.TAG_PAREN_RE[1], KPIEDGARScenario.TAG_PAREN_ESC[1], tmp0)
+        return tmp1
+
+    @staticmethod
+    def get_output_text(
+        words: List[str],
+        annotations: List[Dict],
+    ) -> str:
+        # def get_entity_for_annotation(words: List[str], annotation: Dict) -> str
+        entities: List[str] = []
+        for annotation in annotations:
+            annotation_type = annotation["type_"]
+            if annotation_type not in KPIEDGARScenario.TAG_DICT:
+                continue
+            start_idx = annotation["start"]
+            end_idx = annotation["end"]
+            annotated_words = words[start_idx:end_idx]
+            phrase = KPIEDGARScenario.escape_parenthesis(" ".join(annotated_words))
+            entities.append(
+                "%s %s%s%s" % (phrase, KPIEDGARScenario.TAG_PAREN[0], annotation_type, KPIEDGARScenario.TAG_PAREN[1])
+            )
+
+        return ", ".join(entities)
+
+    @staticmethod
+    def sentences_to_instances(sentences: List[Dict]) -> List[Instance]:
+        instances: List[Instance] = []
+        for sentence in sentences:
+            dataset_split: str = sentence["split_type"]
+            if dataset_split is None:
+                continue
+            split = KPIEDGARScenario.DATASET_SPLIT_TO_HELM_SPLIT[dataset_split]
+
+            words: List[str] = [word_dict["value"] for word_dict in sentence["words"]]
+            passage = KPIEDGARScenario.escape_parenthesis(" ".join(words))
+            input_text = (
+                "Context: %s\n"
+                "Task: Extract key performance indicators (KPIs) and values from the above text. Also, specify one of the following categories to each of the extracted KPIs and values in brackets.\n"  # noqa: E501
+                "%s" % (passage, KPIEDGARScenario.TAG_DESCRIPTIONS)
+            )
+
+            annotations = sentence["entities_anno"]
+            output_text = KPIEDGARScenario.get_output_text(words, annotations)
+            if not output_text:
+                continue
+
+            instances.append(
+                Instance(
+                    input=Input(text=input_text),
+                    references=[Reference(Output(text=output_text), tags=[CORRECT_TAG])],
+                    split=split,
+                )
+            )
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = os.path.join(output_path, "data")
+        ensure_directory_exists(data_path)
+
+        base_url = self.JSON_URL
+        dataset_file_name = self.JSON_FILENAME
+        target_path = os.path.join(data_path, dataset_file_name)
+        ensure_file_downloaded(source_url=base_url, target_path=target_path)
+
+        with open(target_path, "r") as f:
+            raw_dataset = json.load(f)
+        return KPIEDGARScenario.sentences_to_instances(KPIEDGARScenario.get_sentences(raw_dataset))

helm/benchmark/scenarios/med_dialog_scenario.py

@@ -90,7 +90,12 @@ class MedDialogScenario(Scenario):
     """
 
     name = "med_dialog"
-    description = "A collection of doctor-patient conversations with corresponding summaries."
+    description = (
+        "MedDialog is a benchmark of real-world doctor-patient conversations focused on health-related"
+        "concerns and advice. Each dialogue is paired with a one-sentence summary"
+        "that reflects the core patient question or exchange. The benchmark evaluates a model's"
+        "ability to condense medical dialogue into concise, informative summaries."
+    )
     tags = ["dialogue", "biomedical"]
 
     def __init__(self, subset: str):

helm/benchmark/scenarios/medalign_scenario.py

@@ -60,12 +60,18 @@ class MedalignScenario(Scenario):
     """
 
     name = "medalign"
-    description = "A dataset that asks models to answer questions/follow instructions over longitudinal EHR."
+    description = (
+        "MedAlign is a benchmark that evaluates a model's ability to interpret and follow"
+        "instructions grounded in longitudinal electronic health records (EHR). Each instance"
+        "includes an event-stream style patient record and a natural language question or task,"
+        "requiring clinically informed reading comprehension and reasoning."
+    )
     tags = ["knowledge", "reasoning", "biomedical"]
 
-    def __init__(self, max_length: int):
+    def __init__(self, max_length: int, data_path: str):
         super().__init__()
         self.max_length = max_length
+        self.data_path = data_path
 
     def process_tsv(self, data) -> List[Instance]:
         instances: List[Instance] = []
@@ -84,5 +90,5 @@ class MedalignScenario(Scenario):
         return instances
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        dataset = return_dataset_dataframe(self.max_length)
+        dataset = return_dataset_dataframe(self.max_length, self.data_path)
         return self.process_tsv(dataset)

helm/benchmark/scenarios/medalign_scenario_helper.py

@@ -19,6 +19,7 @@ from typing import Any, Dict, Optional, Union, Callable
 from langchain.schema import Document
 import langchain_community
 
+from helm.common.general import check_file_exists
 
 
 def get_instructions(path_to_instructions: str) -> Dict[int, Dict[str, Any]]:
@@ -399,19 +400,21 @@ def add_reference_responses(prompts_df, path_to_reference_responses) -> pd.DataF
     Returns:
     pd.DataFrame: DataFrame containing the processed data.
     """
-    gold_df = pd.read_csv(path_to_reference_responses)
+    gold_df = pd.read_csv(path_to_reference_responses, sep='\t')
     gold_df = gold_df.query("annotator_num == 'Annotator_1'")
     gold_df = gold_df[["instruction_id", "clinician_response"]]
     merged_df = gold_df.merge(prompts_df, on="instruction_id", how="inner")
     return merged_df
 
 
-def return_dataset_dataframe(max_length: int) -> pd.DataFrame:
+def return_dataset_dataframe(max_length: int, data_path: str) -> pd.DataFrame:
     target_context_length = max_length
     generation_length = 256
-    path_to_instructions = "/share/pi/nigam/datasets/medalign_release_fixes/clinician-reviewed-model-responses.tsv"
-    path_to_ehrs = "/share/pi/nigam/datasets/medalign_release_fixes/medalign_ehr_xml"
-    path_to_reference_responses = "/share/pi/nigam/scottyf/clinician-instruction-responses.csv"
+    path_to_instructions = os.path.join(data_path, "clinician-reviewed-model-responses.tsv")
+    check_file_exists(path_to_instructions, msg=f"[MedAlignScenario] Required instructions file not found: '{path_to_instructions}'")
+    path_to_ehrs = os.path.join(data_path, "medalign_ehr_xml")
+    path_to_reference_responses = os.path.join(data_path, "clinician-instruction-responses.tsv")
+    check_file_exists(path_to_reference_responses, msg=f"[MedAlignScenario] Required clinician responses file not found: '{path_to_reference_responses}'")
     use_RAG = False
     include_ehr = True
     tokenizer = "tiktoken"

helm/benchmark/scenarios/medbullets_scenario.py

@@ -67,8 +67,13 @@ class MedBulletsScenario(Scenario):
         "https://raw.githubusercontent.com/HanjieChen/ChallengeClinicalQA/refs/heads/main/medbullets/"
     )
 
-    name = "medbullet"
-    description = "A USMLE-style medical question dataset with multiple-choice answers and explanations."
+    name = "medbullets"
+    description = (
+        "Medbullets is a benchmark of USMLE-style medical questions designed to assess a"
+        "model’s ability to understand and apply clinical knowledge. Each question is accompanied"
+        "by a patient scenario and five multiple-choice options, similar to those found on"
+        "Step 2 and Step 3 on the US medical licensing exam."
+    )
     tags = ["reasoning", "biomedical"]
 
     # Define the possible answer choices

helm/benchmark/scenarios/medcalc_bench_scenario.py

@@ -71,8 +71,10 @@ class MedCalcBenchScenario(Scenario):
 
     name = "medcalc_bench"
     description = (
-        "A dataset which consists of a patient note, a question "
-        "requesting to compute a specific medical value, and a ground truth answer."
+        "MedCalc-Bench is a benchmark designed to evaluate models on their ability to compute"
+        "clinically relevant values from patient notes. Each instance consists of a clinical note"
+        "describing the patient's condition, a diagnostic question targeting a specific medical"
+        "value, and a ground truth response."
     )
     tags = ["knowledge", "reasoning", "biomedical"]
 

helm/benchmark/scenarios/medec_scenario.py

@@ -50,7 +50,12 @@ class MedecScenario(Scenario):
     TEST_URL = f"https://raw.githubusercontent.com/abachaa/MEDEC/{GIT_HASH}/MEDEC-MS/MEDEC-MS-TestSet-with-GroundTruth-and-ErrorType.csv"  # noqa: E501
 
     name = "medec"
-    description = "A dataset containing medical narratives with error detection and correction pairs."
+    description = (
+        "Medec is a benchmark composed of clinical narratives that include either correct"
+        "documentation or medical errors. Each entry includes sentence-level identifiers and an"
+        "associated correction task. The model must review the narrative and either identify"
+        "the erroneous sentence and correct it, or confirm that the text is entirely accurate."
+    )
     tags = ["error_detection", "error_correction", "biomedical"]
 
     def download_csv(self, url: str, output_path: str, file_name: str) -> str:

helm/benchmark/scenarios/medhallu_scenario.py

@@ -20,7 +20,13 @@ class MedHalluScenario(Scenario):
     """
 
     name = "medhallu"
-    description = "A dataset of PubMed articles and associated questions, with the objective being to classify whether the answer is factual or hallucinated."  # noqa: E501
+    description = (
+        "MedHallu is a benchmark focused on evaluating factual correctness in biomedical"
+        "question answering. Each instance contains a PubMed-derived knowledge snippet, a"
+        "biomedical question, and a model-generated answer. The task is to classify whether the"
+        "answer is factually correct or contains hallucinated (non-grounded) information. This"
+        "benchmark is designed to assess the factual reliability of medical language models."
+    )
     tags = ["knowledge", "reasoning", "biomedical"]
 
     def create_instance(self, question, knowledge, answer, label, split):

helm/benchmark/scenarios/medi_qa_scenario.py

@@ -49,9 +49,11 @@ class MediQAScenario(Scenario):
 
     name = "medi_qa"
     description = (
-        "A dataset including a medical question, a set of candidate answers,"
-        "relevance annotations for ranking, and additional context to evaluate understanding"
-        "and retrieval capabilities in a healthcare setting."
+        "MEDIQA is a benchmark designed to evaluate a model's ability to retrieve and generate"
+        "medically accurate answers to patient-generated questions. Each instance includes a"
+        "consumer health question, a set of candidate answers (used in ranking tasks), relevance"
+        "annotations, and optionally, additional context. The benchmark focuses on supporting"
+        "patient understanding and accessibility in health communication."
     )
     tags = ["knowledge", "biomedical"]
 
@@ -88,7 +90,11 @@ class MediQAScenario(Scenario):
 
     def get_instances(self, output_path: str) -> List[Instance]:
         # Load the MEDIQA dataset from Hugging Face
-        dataset = load_dataset("bigbio/mediqa_qa")
+        dataset = load_dataset(
+            "bigbio/mediqa_qa",
+            trust_remote_code=True,
+            revision="9288641f4c785c95dc9079fa526dabb12efdb041",
+        )
 
         # Process all the instances
         instances: List[Instance] = []

helm/benchmark/scenarios/medication_qa_scenario.py

@@ -31,7 +31,13 @@ class MedicationQAScenario(Scenario):
     FILENAME = "MedInfo2019-QA-Medications.xlsx"
 
     name = "medication_qa"
-    description = "Open text question-answer pairs regarding consumer health questions about medication."
+    description = (
+        "MedicationQA is a benchmark composed of open-ended consumer health questions"
+        "specifically focused on medications. Each example consists of a free-form question"
+        "and a corresponding medically grounded answer. The benchmark evaluates a model's"
+        "ability to provide accurate, accessible, and informative medication-related responses"
+        "for a lay audience."
+    )
     tags = ["knowledge", "generation", "question_answering", "biomedical"]
 
     def download_medication_qa(self, path: str):

helm/benchmark/scenarios/melt_ir_scenario.py

@@ -0,0 +1,171 @@
+from typing import List, Optional
+
+from datasets import load_dataset, Dataset
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    make_rank_tag,
+)
+
+
+class MELTInformationRetrievalScenario(Scenario):
+    name = "melt_information_retrieval"
+    description = "Scenario for information retrieval tasks."
+    tags = ["information_retrieval"]
+
+    """ Dictionary mapping task track tuples to the number of queries. """
+    NUM_TRAIN_QUERIES = 1000
+
+    """ Upper and lower bounds on top-k.
+
+    The top-k number represents the number of passages we will consider per
+    query. Max top-k for the train and validation files are set to the number
+    of passages included in the corresponding top-k files.
+    """
+    MIN_TOPK: int = 11
+    MAX_TRAIN_TOPK: int = 20
+    MAX_VALID_TOPK: int = 1000
+
+    def __init__(
+        self, dataset_name: str, revision: str, subset: Optional[str] = None, valid_topk: Optional[int] = None
+    ):
+        """The constructor for the MSMARCOScenario.
+
+        Args:
+            dataset_name: The name of the dataset.
+            revision: The revision of the dataset to use.
+            subset: The subset of the dataset to use. Defaults to "".
+            valid_topk: If set, specifies the number of top documents for which the
+                validation instances will be created. Must be in the range
+                [self.MIN_TOPK, self.MAX_VALID_TOPK].
+        """
+        super().__init__()
+
+        # Input validation
+        self.dataset_name = dataset_name
+        self.revision = revision
+        self.subset = subset
+        self.valid_topk: Optional[int] = valid_topk
+        if self.valid_topk is not None:
+            assert valid_topk and self.MIN_TOPK <= valid_topk <= self.MAX_VALID_TOPK
+
+    def get_train_instances(self) -> List[Instance]:
+        """Get training instances.
+        References for each instance are selected as follows:
+            1. We select 1 correct reference, where the documents included
+            corresponds to the best document for the given train query.
+            2. We create 1 wrong reference, where the document included
+            corresponds to a non-gold document for the given train query.
+        """
+        dataset = load_dataset(
+            self.dataset_name,
+            self.subset,
+            revision=self.revision,
+            trust_remote_code=True,
+        )
+        instances = []
+        for i, sample in enumerate(dataset["train"]):
+
+            if i >= self.NUM_TRAIN_QUERIES:
+                break
+
+            references = [
+                Reference(Output(text=sample["positive"]), tags=[CORRECT_TAG]),
+                Reference(Output(text=sample["negative"]), tags=[]),
+            ]
+
+            instances.append(Instance(Input(text=sample["query"]), references=references, split=TRAIN_SPLIT))
+        return instances
+
+    def get_valid_instances(self) -> List[Instance]:
+        """Get validation instances.
+        By default, we create a reference for each Document ID for which there
+        is a judgment with respect to the provided Query ID.
+
+        If self.valid_topk is not None, we ensure that a reference is created
+        for all the documents that appear in top self.valid_topk documents for
+        the given validation query.
+        """
+        dataset = load_dataset(
+            self.dataset_name,
+            f"runs-{self.subset}",
+            revision=self.revision,
+            trust_remote_code=True,
+        )
+        instances = []
+        for sample in dataset["bm25"]:
+            references = []
+
+            for k, passage_dict in enumerate(Dataset.from_dict(sample["passages"])):
+                if self.valid_topk is None or k >= self.valid_topk:
+                    break
+                tags = []
+                tags.append(f"docid={passage_dict['id']}")
+                if k == 0:
+                    tags.append(CORRECT_TAG)
+                tags.append(make_rank_tag(rank=k + 1))  # Top-k rank
+                references.append(Reference(Output(text=passage_dict["passage"]), tags=tags))
+
+            instances.append(Instance(Input(text=sample["query"]), references=references, split=VALID_SPLIT))
+
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Get instances for this scenario.
+
+        Refer to the documentation of the following methods for details on how
+        the instances are created:
+            * self.get_train_instances
+            * self.get_valid_instances
+        """
+
+        hlog("Preparing training instances.")
+        train_instances = self.get_train_instances()
+
+        hlog("Preparing validation instances.")
+        valid_instances = self.get_valid_instances()
+
+        return train_instances + valid_instances
+
+
+class MELTInformationRetrievalMMARCOScenario(MELTInformationRetrievalScenario):
+    """
+    Scenario for the MMARCO dataset.
+    """
+
+    name = "melt_information_retrieval_mmarco"
+    description = "MMARCO dataset for information retrieval in Vietnamese."
+    tags = ["information_retrieval"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="unicamp-dl/mmarco",
+            revision="6d039c4638c0ba3e46a9cb7b498b145e7edc6230",
+            subset="vietnamese",
+            **kwargs,
+        )
+
+
+class MELTInformationRetrievalMRobustScenario(MELTInformationRetrievalScenario):
+    """
+    Scenario for the MRobust dataset.
+    """
+
+    name = "melt_information_retrieval_mrobust"
+    description = "MRobust dataset for information retrieval in Vietnamese."
+    tags = ["information_retrieval"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="unicamp-dl/mrobust",
+            revision="fda452a7fbfd9550db2f78d9d98e6b3ec16734df",
+            subset="vietnamese",
+            **kwargs,
+        )