crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/math_scenario.py

@@ -1,19 +1,33 @@
 import collections
+import os
 import typing
 from typing import Dict, List, Optional
 from datasets import load_dataset, DatasetDict
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 def remove_boxed(string: str) -> Optional[str]:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
-    Extract the text within a \\boxed{...} environment.
+    Extract the text within a \boxed{...} environment.
 
     Example:
-    >>> remove_boxed(\\boxed{\\frac{2}{3}})
-    \\frac{2}{3}
+        >>> from helm.benchmark.scenarios.math_scenario import *  # NOQA
+        >>> remove_boxed(r'\boxed{\frac{2}{3}}')
+        '\\frac{2}{3}'
     """
     left = "\\boxed{"
     try:
@@ -57,17 +71,17 @@ def last_boxed_only_string(string: str) -> Optional[str]:
 
 
 def _fix_fracs(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat fractions.
 
     Examples:
-    >>> _fix_fracs("\\frac1b")
-    \frac{1}{b}
-    >>> _fix_fracs("\\frac12")
-    \frac{1}{2}
-    >>> _fix_fracs("\\frac1{72}")
-    \frac{1}{72}
+        >>> _fix_fracs(r"\frac1b")
+        '\\frac{1}{b}'
+        >>> _fix_fracs(r"\frac12")
+        '\\frac{1}{2}'
+        >>> _fix_fracs(r"\frac1{72}")
+        '\\frac{1}{72}'
     """
     substrs = string.split("\\frac")
     new_str = substrs[0]
@@ -101,13 +115,13 @@ def _fix_fracs(string: str) -> str:
 
 
 def _fix_a_slash_b(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat fractions formatted as a/b to \\frac{a}{b}.
 
     Example:
-    >>> _fix_a_slash_b("2/3")
-    \frac{2}{3}
+        >>> _fix_a_slash_b(r"2/3")
+        '\\frac{2}{3}'
     """
     if len(string.split("/")) != 2:
         return string
@@ -138,13 +152,13 @@ def _remove_right_units(string: str) -> str:
 
 
 def _fix_sqrt(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat square roots.
 
     Example:
-    >>> _fix_sqrt("\\sqrt3")
-    \sqrt{3}
+        >>> _fix_sqrt("\\sqrt3")
+        '\\sqrt{3}'
     """
     if "\\sqrt" not in string:
         return string
@@ -199,7 +213,7 @@ def _strip_string(string: str) -> str:
 
     # remove percentage
     string = string.replace("\\%", "")
-    string = string.replace("\%", "")
+    string = string.replace(r"\%", "")
 
     # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
     string = string.replace(" .", " 0.")
@@ -354,7 +368,21 @@ class MATHScenario(Scenario):
 
     def get_instances(self, output_path: str) -> List[Instance]:
         dataset = {}
-        data = typing.cast(DatasetDict, load_dataset("competition_math")).sort("problem").shuffle(seed=42)
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        data = (
+            typing.cast(
+                DatasetDict,
+                load_dataset(
+                    "hendrycks/competition_math",
+                    trust_remote_code=True,
+                    cache_dir=cache_dir,
+                    revision="71b758ecc688b2822d07ffa7f8393299f1dc7cac",
+                ),
+            )
+            .sort("problem")
+            .shuffle(seed=42)
+        )
 
         def group_by_key(dataset_list, key):
             dataset_per_key = collections.defaultdict(list)
@@ -366,13 +394,13 @@ class MATHScenario(Scenario):
         for split, split_name in zip([TRAIN_SPLIT, TEST_SPLIT], ["train", "test"]):
             if split == TRAIN_SPLIT and self.use_official_examples:
                 train_instances = [
-                    ("What is $\left(\\frac{7}{8}\\right)^3 \cdot \left(\\frac{7}{8}\\right)^{-3}$?", "1"),
+                    ("What is $\\left(\\frac{7}{8}\\right)^3 \\cdot \\left(\\frac{7}{8}\\right)^{-3}$?", "1"),
                     (
                         "In how many ways can 4 books be selected from a shelf of 6 books"
                         + " if the order in which the books are selected does not matter?",
                         "15",
                     ),
-                    ("Find the distance between the points $(2,1,-4)$ and $(5,8,-3).$", "\sqrt{59}"),
+                    ("Find the distance between the points $(2,1,-4)$ and $(5,8,-3).$", "\\sqrt{59}"),
                     (
                         "The faces of an octahedral die are labeled with digits $1$ through $8$."
                         + " What is the probability, expressed as a common fraction,"
@@ -424,3 +452,34 @@ class MATHScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        taxonomy = TaxonomyInfo(
+            task="numeric answer question answering",
+            what="math competitions (AMC, AIME, etc.)",
+            when="before 2021",
+            who="problem setters",
+            language="synthetic",
+        )
+        if self.use_chain_of_thought:
+            return ScenarioMetadata(
+                name="math_chain_of_thought",
+                display_name="MATH",
+                description="The MATH benchmark for measuring mathematical problem solving on competition "
+                "math problems with chain-of-thought style reasoning [(Hendrycks et al., "
+                "2021)](https://arxiv.org/pdf/2103.03874.pdf).",
+                taxonomy=taxonomy,
+                main_metric="math_equiv_chain_of_thought",
+                main_split="test",
+            )
+        else:
+            return ScenarioMetadata(
+                name="math_regular",
+                display_name="MATH",
+                description="The MATH benchmark for measuring mathematical problem solving on competition "
+                "math problems [(Hendrycks et al., "
+                "2021)](https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/be83ab3ecd0db773eb2dc1b0a17836a1-Abstract-round2.html).",
+                taxonomy=taxonomy,
+                main_metric="math_equiv",
+                main_split="test",
+            )

helm/benchmark/scenarios/mbzuai_human_translated_arabic_mmlu.py

@@ -0,0 +1,68 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MBZUAIHumanTranslatedArabicMMLUScenario(Scenario):
+    """MBZUAI Human-Translated Arabic MMLU
+
+    A translation from MBZUAI by human translators of the Massive Multitask Language Understanding benchmark from this paper:
+
+    - https://arxiv.org/pdf/2009.03300.pdf
+    """  # noqa: E501
+
+    name = "mbzuai_human_translated_arabic_mmlu"
+    description = (
+        "A translation from MBZUAI by human translators of the Massive Multitask Language Understanding benchmark"
+    )
+    tags = ["knowledge", "multiple_choice"]
+
+    def __init__(self, subject: str):
+        super().__init__()
+        self.subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "MBZUAI/human_translated_arabic_mmlu",
+            self.subject,
+            revision="5ed7830fd678cfa6f2d7f0a1a13a4e1a1fa422ac",
+            cache_dir=cache_dir,
+            split="test",
+        )
+        assert isinstance(dataset, datasets.Dataset)
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row_index, row in enumerate(dataset):
+            input = Input(text=row["question"])
+            references: List[Reference] = []
+            for choice_index, choice in enumerate(row["choices"]):
+                references.append(
+                    Reference(
+                        output=Output(text=choice),
+                        tags=[CORRECT_TAG] if choice_index == row["answer"] else [],
+                    )
+                )
+            instance = Instance(
+                id=f"id-{self.subject}-{row_index}",
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/me_q_sum_scenario.py

@@ -2,7 +2,16 @@ import os
 from typing import List
 
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, ALL_SPLITS, CORRECT_TAG, VALID_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    ALL_SPLITS,
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Input,
+    Output,
+)
 
 
 class MeQSumScenario(Scenario):

helm/benchmark/scenarios/med_dialog_scenario.py

@@ -2,8 +2,18 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, ALL_SPLITS, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    ALL_SPLITS,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class MedDialogScenario(Scenario):
@@ -91,9 +101,10 @@ class MedDialogScenario(Scenario):
 
     name = "med_dialog"
     description = (
-        "The MedDialog dataset (English) contains conversations between doctors and patients. "
-        "It has 0.26 million dialogues. The data is continuously growing and more dialogues will be added. "
-        "The raw dialogues are from healthcaremagic.com and icliniq.com."
+        "MedDialog is a benchmark of real-world doctor-patient conversations focused on health-related"
+        "concerns and advice. Each dialogue is paired with a one-sentence summary"
+        "that reflects the core patient question or exchange. The benchmark evaluates a model's"
+        "ability to condense medical dialogue into concise, informative summaries."
     )
     tags = ["dialogue", "biomedical"]
 
@@ -109,24 +120,47 @@ class MedDialogScenario(Scenario):
         instances: List[Instance] = []
 
         for split in ALL_SPLITS:
-            split_file_name: str = f"{split}.json"
-            split_path: str = os.path.join(data_path, split_file_name)
-            ensure_file_downloaded(
-                source_url="https://worksheets.codalab.org/rest/bundles/0x82f0c47f6d3e4462ae9ef8ea39eebe64/"
-                f"contents/blob/{self.subset}/{split_file_name}",
-                target_path=split_path,
-                unpack=False,
-            )
-
-            with open(split_path, "r") as f:
-                examples: List = json.load(f)["data"]
-                for example in examples:
-                    instances.append(
-                        Instance(
-                            input=Input(text=example["src"]),
-                            references=[Reference(Output(text=example["tgt"]), tags=[CORRECT_TAG])],
-                            split=split,
+            # Limit to zero shot setting
+            if split == "test":
+                split_file_name: str = f"{split}.json"
+                split_path: str = os.path.join(data_path, split_file_name)
+                ensure_file_downloaded(
+                    source_url="https://worksheets.codalab.org/rest/bundles/0x82f0c47f6d3e4462ae9ef8ea39eebe64/"
+                    f"contents/blob/{self.subset}/{split_file_name}",
+                    target_path=split_path,
+                    unpack=False,
+                )
+
+                with open(split_path, "r") as f:
+                    examples: List = json.load(f)["data"]
+                    for example in examples:
+                        instances.append(
+                            Instance(
+                                input=Input(text=example["src"]),
+                                references=[Reference(Output(text=example["tgt"]), tags=[CORRECT_TAG])],
+                                split=split,
+                            )
                         )
-                    )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="med_dialog",
+            display_name="MedDialog",
+            short_display_name="MedDialog",
+            description="MedDialog is a benchmark of real-world doctor-patient conversations focused on "
+            "health-related concerns and advice. Each dialogue is paired with a "
+            "one-sentence summary that reflects the core patient question or exchange. The "
+            "benchmark evaluates a model's ability to condense medical dialogue into "
+            "concise, informative summaries.",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate summaries of doctor-patient conversations",
+                when="Any",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="med_dialog_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/med_mcqa_scenario.py

@@ -2,8 +2,19 @@ import json
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TRAIN_SPLIT, VALID_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class MedMCQAScenario(Scenario):
@@ -100,3 +111,15 @@ class MedMCQAScenario(Scenario):
                     instances.append(instance)
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="med_mcqa",
+            display_name="MedMCQA",
+            description='MedMCQA is a "multiple-choice question answering (MCQA) dataset designed to '
+            "address real-world medical entrance exam questions ([Flores et al. "
+            "2020](https://arxiv.org/abs/2203.14371)).",
+            taxonomy=TaxonomyInfo(task="question answering", what="n/a", when="n/a", who="n/a", language="English"),
+            main_metric="exact_match",
+            main_split="valid",
+        )

helm/benchmark/scenarios/med_paragraph_simplification_scenario.py

@@ -2,7 +2,16 @@ import os
 from typing import List
 
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, ALL_SPLITS, CORRECT_TAG, VALID_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    ALL_SPLITS,
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Input,
+    Output,
+)
 
 
 class MedParagraphSimplificationScenario(Scenario):

helm/benchmark/scenarios/med_qa_scenario.py

@@ -2,8 +2,19 @@ import json
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, ALL_SPLITS, CORRECT_TAG, VALID_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    ALL_SPLITS,
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class MedQAScenario(Scenario):
@@ -94,3 +105,21 @@ class MedQAScenario(Scenario):
                     instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="med_qa",
+            display_name="MedQA",
+            description="MedQA is an open domain question answering dataset composed of questions from "
+            "professional medical board exams ([Jin et al. "
+            "2020](https://arxiv.org/pdf/2009.13081.pdf)).",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="US medical licensing exams",
+                when="before 2020",
+                who="problem setters",
+                language="English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/medalign_scenario.py

@@ -0,0 +1,117 @@
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    PassageQuestionInput,
+    Output,
+    ScenarioMetadata,
+)
+from helm.benchmark.scenarios.medalign_scenario_helper import return_dataset_dataframe  # type: ignore
+
+
+class MedalignScenario(Scenario):
+    """
+    Scenario defining the MedAlign task as defined in the following work by Fleming et al:
+    @article{fleming2023medalign,
+      title={MedAlign: A Clinician-Generated Dataset for Instruction Following with Electronic Medical Records},
+      author={Scott L. Fleming
+        and Alejandro Lozano
+        and William J. Haberkorn
+        and Jenelle A. Jindal
+        and Eduardo P. Reis
+        and Rahul Thapa
+        and Louis Blankemeier
+        and Julian Z. Genkins
+        and Ethan Steinberg
+        and Ashwin Nayak
+        and Birju S. Patel
+        and Chia-Chun Chiang
+        and Alison Callahan
+        and Zepeng Huo
+        and Sergios Gatidis
+        and Scott J. Adams
+        and Oluseyi Fayanju
+        and Shreya J. Shah
+        and Thomas Savage
+        and Ethan Goh
+        and Akshay S. Chaudhari
+        and Nima Aghaeepour
+        and Christopher Sharp
+        and Michael A. Pfeffer
+        and Percy Liang
+        and Jonathan H. Chen
+        and Keith E. Morse
+        and Emma P. Brunskill
+        and Jason A. Fries
+        and Nigam H. Shah},
+      journal={arXiv preprint arXiv:2308.14089},
+      year={2023}
+    }
+    Each instance includes:
+    - input: the instruction and patient record
+    - reference: the clinical 'gold standard' completion for the instruction for the given patient record
+    This is a clinical instruction-following task, wherein a generative language model must follow
+    the instructions using the provided patient record. As explained in the MedAlign work, each example
+    is guaranteed to be completable for the given patient record.
+    This task is evaluated using COMET and BERTScore metrics.
+    """
+
+    name = "medalign"
+    description = (
+        "MedAlign is a benchmark that evaluates a model's ability to interpret and follow"
+        "instructions grounded in longitudinal electronic health records (EHR). Each instance"
+        "includes an event-stream style patient record and a natural language question or task,"
+        "requiring clinically informed reading comprehension and reasoning."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    def __init__(self, max_length: int, data_path: str):
+        super().__init__()
+        self.max_length = max_length
+        self.data_path = data_path
+
+    def process_tsv(self, data) -> List[Instance]:
+        instances: List[Instance] = []
+        for index, row in data.iterrows():
+            question = row["prompt"]
+            ground_truth_answer = row["clinician_response"]
+
+            prompt = PassageQuestionInput(passage="", question=question)
+
+            instance = Instance(
+                input=prompt,
+                references=[Reference(Output(text=ground_truth_answer), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset = return_dataset_dataframe(self.max_length, self.data_path)
+        return self.process_tsv(dataset)
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medalign",
+            display_name="MedAlign",
+            short_display_name="MedAlign",
+            description="MedAlign is a benchmark that evaluates a model's ability to interpret and "
+            "follow instructions grounded in longitudinal electronic health records (EHR). "
+            "Each instance includes an event-stream style patient record and a natural "
+            "language question or task, requiring clinically informed reading comprehension "
+            "and reasoning [(Fleming et al., 2023)](https://arxiv.org/abs/2308.14089).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Answer questions and follow instructions over longitudinal EHR",
+                when="Any",
+                who="Clinician, Researcher",
+                language="English",
+            ),
+            main_metric="medalign_accuracy",
+            main_split="test",
+        )