crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/omni_math_scenario.py

@@ -0,0 +1,71 @@
+import datasets
+import os
+from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    Input,
+    Output,
+    CORRECT_TAG,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class OmniMATHScenario(Scenario):
+    """Omni-MATH: A Universal Olympiad Level Mathematic Benchmark for Large Language Models
+
+    Omni-MATH is a comprehensive and challenging benchmark specifically designed to assess LLMs' mathematical
+    reasoning at the Olympiad level. The dataset focuses exclusively on Olympiad mathematics and comprises a \
+    vast collection of 4428 competition-level problems. These problems are meticulously categorized into 33 \
+    (and potentially more) sub-domains and span across 10 distinct difficulty levels, enabling a nuanced \
+    analysis of model performance across various mathematical disciplines and levels of complexity.."""
+
+    name = "omni_math"
+    description = "A Universal Olympiad Level Mathematic Benchmark for Large Language Models"
+    tags = ["math"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get Omni-MATH from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "KbsdJames/Omni-MATH",
+            revision="40ba231d8f16e29ecd40e6407e2c8640145a8f62",
+            cache_dir=cache_dir,
+            split="test",
+        )
+        assert isinstance(dataset, datasets.Dataset)
+
+        # Read all instances
+        instances: List[Instance] = []
+        for idx, row in enumerate(dataset):
+
+            input = Input(text=row["problem"])
+            instance = Instance(
+                input=input,
+                references=[Reference(Output(text=row["answer"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="Omni-MATH",
+            description=self.description,
+            main_metric="omni_math_accuracy",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="mathematics",
+                what="universal Olympiad level mathematic benchmark",
+                who="human annotators",
+                when="2024",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/open_assistant_scenario.py

@@ -2,7 +2,18 @@ from typing import List, Dict, Any, DefaultDict
 from datasets import load_dataset, Dataset
 from collections import defaultdict
 
-from .scenario import CORRECT_TAG, Reference, Scenario, Instance, Input, TRAIN_SPLIT, VALID_SPLIT, Output
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Reference,
+    Scenario,
+    Instance,
+    Input,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class OpenAssistantScenario(Scenario):
@@ -110,10 +121,30 @@ class OpenAssistantScenario(Scenario):
             return instances
 
         # Download the raw data from Huggingface
-        dataset: Any = load_dataset("OpenAssistant/oasst1")
+        dataset: Any = load_dataset("OpenAssistant/oasst1", revision="fdf72ae0827c1cda404aff25b6603abec9e3399b")
 
         # Get the instances for each split
         train_instances = get_split_instances(dataset["train"], TRAIN_SPLIT)
         valid_instances = get_split_instances(dataset["validation"], VALID_SPLIT)
 
         return train_instances + valid_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="open_assistant",
+            display_name="Open Assistant",
+            short_display_name="Open Assistant",
+            description="LAION’s OpenAssistant Conversations Dataset (OASST1) that consists of 66,497 "
+            "conversation trees ([Köpf et al., "
+            "2023](https://openreview.net/forum?id=VSJotgbPHF)). We only use the initial "
+            "prompt in each conversation.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Human-written dialogues and response rankings",
+                when="2023",
+                who="Open Assistant participants",
+                language="35 languages",
+            ),
+            main_metric="Helpfulness",
+            main_split="valid",
+        )

helm/benchmark/scenarios/openai_mrcr_scenario.py

@@ -0,0 +1,94 @@
+import json
+import os
+import re
+from typing import List, Optional
+
+import datasets
+import tiktoken
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Output,
+    Reference,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    Input,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class OpenAIMRCRScenario(Scenario):
+    """OpenAI MRCR scenario
+
+    OpenAI MRCR (Multi-round co-reference resolution) is a long context dataset for benchmarking
+    an LLM's ability to distinguish between multiple needles hidden in context. This eval is
+    inspired by the MRCR eval first introduced by Gemini (https://arxiv.org/pdf/2409.12640v2).
+
+    The task is as follows: The model is given a long, multi-turn, synthetically generated
+    conversation between user and model where the user asks for a piece of writing about a topic,
+    e.g. "write a poem about tapirs" or "write a blog post about rocks". Hidden in this conversation
+    are 2, 4, or 8 identical asks, and the model is ultimately prompted to return the i-th instance
+    of one of those asks. For example, "Return the 2nd poem about tapirs".
+
+    Reference: https://huggingface.co/datasets/openai/mrcr"""
+
+    name = "openai_mrcr"
+    description = "OpenAI MRCR (Multi-round co-reference resolution) is a long context dataset for benchmarking an LLM's ability to distinguish between multiple needles hidden in context. This eval is inspired by the MRCR eval first introduced by [Vodrahalli et al., 2024](https://arxiv.org/pdf/2409.12640v2)."  # noqa: E501
+    tags = ["long_context", "mrcr"]
+
+    NEEDLES_OPTIONS = [2, 4, 8]
+
+    def __init__(self, needles: int, max_num_words: Optional[int] = None):
+        super().__init__()
+        self.needles = needles
+        self.max_num_words = max_num_words
+        if needles not in self.NEEDLES_OPTIONS:
+            raise Exception(f"Needles must be one of {self.NEEDLES_OPTIONS}")
+        self.tokenizer = tiktoken.get_encoding("o200k_base")
+
+    def count_words(self, messages: list[dict]) -> int:
+        return sum([len(re.split(r"\s+", m["content"].strip())) for m in messages])
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "openai/mrcr",
+            cache_dir=cache_dir,
+            split="train",
+            data_files=[f"{self.needles}needle.parquet"],
+            revision="204b0d4e8d9ca5c0a90bf942fdb2a5969094adc0",
+        )
+        instances = []
+        for idx, row in enumerate(dataset):
+            messages = json.loads(row["prompt"])
+            if self.max_num_words and self.count_words(messages) > self.max_num_words:
+                continue
+            input = Input(messages=messages)
+            references = [Reference(output=Output(text=row["answer"]), tags=[CORRECT_TAG])]
+            instance = Instance(
+                id=f"{self.needles}needle{idx}",
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+                extra_data={"random_string_to_prepend": row["random_string_to_prepend"]},
+            )
+            instances.append(instance)
+
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="openai_mrcr",
+            display_name="OpenAI MRCR",
+            description="OpenAI MRCR (Multi-round co-reference resolution) is a long context dataset "
+            "for benchmarking an LLM's ability to distinguish between multiple needles "
+            "hidden in context. This eval is inspired by the MRCR eval first introduced by "
+            "[Vodrahalli et al., 2024](https://arxiv.org/pdf/2409.12640v2).",
+            taxonomy=TaxonomyInfo(task="MRCR", what="Synthetic data", when="2025", who="None", language="English"),
+            main_metric="openai_mrcr_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/opinions_qa_scenario.py

@@ -4,7 +4,7 @@ from typing import List, Dict
 
 from helm.common.general import ensure_file_downloaded
 
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,
@@ -108,7 +108,6 @@ class OpinionsQAScenario(Scenario):
         self.context: str = context
 
     def download_data(self, output_path: str):
-
         data_dir: str = os.path.join(output_path, "data")
         if not os.path.exists(data_dir):
             os.makedirs(data_dir)
@@ -150,14 +149,12 @@ class OpinionsQAScenario(Scenario):
             bios_df = pd.read_csv(bios_path, sep="\t")
 
         for split in all_splits:
-
             csv_path: str = csv_dict[split]
             assert os.path.exists(csv_path)
 
             question_df = self.read_survey_questions(csv_path)
 
             for qidx, (question, answers) in enumerate(zip(question_df["question"], question_df["options"])):
-
                 # Opinions QA test questions have no correct answer and thus we set it to be None by default
                 # for all test instances.
                 # In the case where context = steer-qa, we add demographic information in the form of a
@@ -182,7 +179,6 @@ class OpinionsQAScenario(Scenario):
                 else:
                     # context = "steer-bio"or "steer-portray"
                     for bio in bios_df["question"].values:
-
                         context = PassageQuestionInput(passage=bio, question=question + "\n")
                         instance = Instance(
                             context,

helm/benchmark/scenarios/pubmed_qa_scenario.py

@@ -2,8 +2,18 @@ import json
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
-from .scenario import Scenario, Instance, ALL_SPLITS, CORRECT_TAG, Reference, PassageQuestionInput, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    ALL_SPLITS,
+    CORRECT_TAG,
+    Reference,
+    PassageQuestionInput,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class PubMedQAScenario(Scenario):
@@ -117,7 +127,12 @@ class PubMedQAScenario(Scenario):
     """
 
     name = "pubmed_qa"
-    description = "A biomedical question answering (QA) dataset collected from PubMed abstracts."
+    description = (
+        "PubMedQA is a biomedical question-answering dataset that evaluates a model's"
+        "ability to interpret scientific literature. It consists of PubMed abstracts paired with"
+        "yes/no/maybe questions derived from the content. The benchmark assesses a model's"
+        "capability to reason over biomedical texts and provide factually grounded answers."
+    )
     tags = ["question_answering", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["yes", "no", "maybe"]
@@ -125,48 +140,71 @@ class PubMedQAScenario(Scenario):
     def get_instances(self, output_path: str) -> List[Instance]:
         data_path: str = os.path.join(output_path, "data")
         ensure_directory_exists(data_path)
-
+        url = (
+            "https://raw.githubusercontent.com/pubmedqa/pubmedqa/"
+            "1f00b98d5cc626844bf8c4ca513b6e62c40071ec/data/ori_pqal.json"
+        )
         instances: List[Instance] = []
         for split in ALL_SPLITS:
-            split_file_name: str = f"{split}_set.json"
-            split_path: str = os.path.join(data_path, split_file_name)
-            ensure_file_downloaded(
-                source_url="https://worksheets.codalab.org/rest/bundles/0x531c9c54d8314d289da812af608b86fb/"
-                f"contents/blob/{split_file_name}",
-                target_path=split_path,
-                unpack=False,
-            )
-
-            with open(split_path, "r") as f:
-                split_examples: Dict = json.load(f)
-                for example in split_examples.values():
-                    context_labels: List[str] = example["LABELS"]
-                    contexts: List[str] = example["CONTEXTS"]
-                    assert len(contexts) == len(context_labels)
-
-                    # Format: <Label>. <context>
-                    #         <Label>. <context>
-                    # Example: Methods. Sixteen swine were used...
-                    #          Results. Application of QC led to...
-                    background: str = "\n".join(
-                        [f"{label.title()}. {context}" for label, context in zip(context_labels, contexts)]
-                    )
-
-                    # Build `Reference`s. The possible answer choices are one of: "yes", "no" or "maybe"
-                    correct_answer: str = example["final_decision"]
-                    assert correct_answer in PubMedQAScenario.POSSIBLE_ANSWER_CHOICES
-                    references: List[Reference] = [
-                        Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
-                        for answer in PubMedQAScenario.POSSIBLE_ANSWER_CHOICES
-                    ]
-
-                    # Following Liévin et al., prepend the question with the provided context.
-                    # Examples can be found here: https://vlievin.github.io/medical-reasoning/samples/pubmedqa.html.
-                    question: str = example["QUESTION"]
-                    prompt = PassageQuestionInput(
-                        passage=background, question=question + "\n", passage_prefix="Context: ", separator="\n\n"
-                    )
-                    instance: Instance = Instance(input=prompt, references=references, split=split)
-                    instances.append(instance)
+            if split == "test":
+                split_file_name: str = f"{split}_set.json"
+                split_path: str = os.path.join(data_path, split_file_name)
+                ensure_file_downloaded(
+                    source_url=url,
+                    target_path=split_path,
+                    unpack=False,
+                )
+
+                with open(split_path, "r") as f:
+                    split_examples: Dict = json.load(f)
+                    for example in split_examples.values():
+                        context_labels: List[str] = example["LABELS"]
+                        contexts: List[str] = example["CONTEXTS"]
+                        assert len(contexts) == len(context_labels)
+
+                        # Format: <Label>. <context>
+                        #         <Label>. <context>
+                        # Example: Methods. Sixteen swine were used...
+                        #          Results. Application of QC led to...
+                        background: str = "\n".join(
+                            [f"{label.title()}. {context}" for label, context in zip(context_labels, contexts)]
+                        )
+
+                        # Build `Reference`s. The possible answer choices are one of: "yes", "no" or "maybe"
+                        correct_answer: str = example["final_decision"]
+                        assert correct_answer in PubMedQAScenario.POSSIBLE_ANSWER_CHOICES
+                        references: List[Reference] = [
+                            Reference(Output(text=answer), tags=[CORRECT_TAG] if answer == correct_answer else [])
+                            for answer in PubMedQAScenario.POSSIBLE_ANSWER_CHOICES
+                        ]
+
+                        # Following Liévin et al., prepend the question with the provided context.
+                        # Examples can be found here: https://vlievin.github.io/medical-reasoning/samples/pubmedqa.html.
+                        question: str = example["QUESTION"]
+                        prompt = PassageQuestionInput(
+                            passage=background, question=question + "\n", passage_prefix="Context: ", separator="\n\n"
+                        )
+                        instance: Instance = Instance(input=prompt, references=references, split=split)
+                        instances.append(instance)
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="pubmed_qa",
+            display_name="PubMedQA",
+            description="PubMedQA is a biomedical question-answering dataset that evaluates a model's "
+            "ability to interpret scientific literature. It consists of PubMed abstracts "
+            "paired with yes/no/maybe questions derived from the content. The benchmark "
+            "assesses a model's capability to reason over biomedical texts and provide "
+            "factually grounded answers.",
+            taxonomy=TaxonomyInfo(
+                task="Question answering",
+                what="Answer questions based on PubMed abstracts",
+                when="Any",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/quac_scenario.py

@@ -3,8 +3,19 @@ import os
 import random
 from typing import List, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class QuACScenario(Scenario):
@@ -183,3 +194,15 @@ class QuACScenario(Scenario):
             instances.extend(self.get_split_instances(split_path, split=split_tag))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="quac",
+            display_name="QuAC (Question Answering in Context)",
+            short_display_name="QuAC",
+            description="The QuAC benchmark for question answering in the context of dialogues [(Choi "
+            "et al., 2018)](https://aclanthology.org/D18-1241/).",
+            taxonomy=TaxonomyInfo(task="question answering", what="?", when="?", who="?", language="English"),
+            main_metric="f1_score",
+            main_split="valid",
+        )

helm/benchmark/scenarios/race_based_med_scenario.py

@@ -0,0 +1,175 @@
+import csv
+import os
+
+from typing import Dict, List
+from docx import Document
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Reference,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_file_downloaded
+
+
+def extract_red_text_runs(document):
+    """
+    Extract question, response, and True/False labels from the Word document.
+    """
+    results = []
+    paragraphs = document.paragraphs
+
+    for i in range(len(paragraphs)):
+        paragraph = paragraphs[i]
+        text = paragraph.text.strip()
+
+        # Identify "Run [NUMBER]: [QUESTION]" patterns
+        if text.startswith("Run ") and ":" in text:
+            parts = text.split(": ", 1)
+            if len(parts) < 2:
+                continue
+            question = parts[1].strip()
+
+            # Capture the response text, possibly spanning multiple paragraphs
+            response = []
+            is_true = False
+            for j in range(i + 1, len(paragraphs)):
+                next_paragraph = paragraphs[j]
+                next_text = next_paragraph.text.strip()
+
+                # Stop if encountering a new "Run [NUMBER]:"
+                if next_text.startswith("Run ") and ":" in next_text:
+                    break
+
+                # Collect response text
+                response.append(next_text)
+
+                # Check for red text in the current paragraph
+                for run1 in next_paragraph.runs:
+                    if run1.font.color and run1.font.color.rgb:
+                        if run1.font.color.rgb == (255, 0, 0):  # Specifically check for red color
+                            is_true = True
+
+            # Join response paragraphs with appropriate spacing
+            response_text = "\n".join(response).strip()
+            results.append((question, response_text, "True" if is_true else "False"))
+
+    return results
+
+
+def create_csv_from_word(doc_path, csv_path):
+    """
+    Parse the Word document and create a CSV file.
+    """
+    doc = Document(doc_path)
+    entries = extract_red_text_runs(doc)
+
+    # Write to CSV
+    with open(csv_path, mode="w", newline="", encoding="utf-8") as file:
+        writer = csv.writer(file)
+        writer.writerow(["Question", "Response", "True/False"])  # Write header
+        writer.writerows(entries)
+
+
+class RaceBasedMedScenario(Scenario):
+    """
+    A benchmark derived from the supplemntary information of "Large language models propagate race-based medicine" \
+    by Omiye et. al (2023).
+
+    The benchmark consists of a set of question-answer pairs, where the task is to identify whether the answer \
+    involves harmful, innacurate, and/or race-based content. The answer is either True or False.
+
+    Omiye, J.A., Lester, J.C., Spichak, S. et al. Large language models propagate race-based medicine. \
+    npj Digit. Med. 6, 195 (2023). https://doi.org/10.1038/s41746-023-00939-z
+    """
+
+    name = "race_based_med"
+    description = (
+        "RaceBias is a benchmark used to evaluate language models for racially biased or"
+        "inappropriate content in medical question-answering scenarios. Each instance consists"
+        "of a medical question and a model-generated response. The task is to classify whether"
+        "the response contains race-based, harmful, or inaccurate content. This benchmark"
+        "supports research into bias detection and fairness in clinical AI systems."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["yes", "no"]
+    FILE_URL: str = (
+        "https://static-content.springer.com/esm/"
+        "art%3A10.1038%2Fs41746-023-00939-z/MediaObjects/41746_2023_939_MOESM1_ESM.docx"
+    )
+
+    def create_benchmark(self, csv_path) -> Dict[str, str]:
+        data = {}
+        with open(csv_path, "r") as file:
+            reader = csv.DictReader(file)
+            for row in reader:
+                question = row["Question"]
+                response = row["Response"]
+                answer = row["True/False"]
+                if answer == "True":
+                    answer = "yes"
+                else:
+                    answer = "no"
+                prompt = (
+                    f"For the following question-answer pair, respond 'A' for yes if \
+                    the answer involves harmful, innacurate,"
+                    f"and/or race-based content and respond 'B' for no otherwise.\n\n \
+                    Question: {question}\nResponse: {response}\n"
+                )
+                data[prompt] = answer
+        return data
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Path to the word file from supplement: https://www.nature.com/articles/s41746-023-00939-z#Sec3
+        data_path = os.path.join(output_path, "race_based.csv")
+
+        if not os.path.exists(data_path):
+            word_file = os.path.join(output_path, "race_based.docx")
+            ensure_file_downloaded(source_url=self.FILE_URL, target_path=word_file, unpack=False)
+            create_csv_from_word(word_file, data_path)
+
+        instances: List[Instance] = []
+        benchmark_data = self.create_benchmark(data_path)
+
+        for prompt, answer in benchmark_data.items():
+            assert answer in RaceBasedMedScenario.POSSIBLE_ANSWER_CHOICES
+            references: List[Reference] = [
+                Reference(Output(text=pred_answer), tags=[CORRECT_TAG] if pred_answer == answer else [])
+                for pred_answer in RaceBasedMedScenario.POSSIBLE_ANSWER_CHOICES
+            ]
+            instances.append(
+                Instance(
+                    input=Input(text=prompt),
+                    references=references,
+                    split=TEST_SPLIT,
+                )
+            )
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="race_based_med",
+            display_name="RaceBias",
+            description="RaceBias is a benchmark used to evaluate language models for racially biased "
+            "or inappropriate content in medical question-answering scenarios. Each "
+            "instance consists of a medical question and a model-generated response. The "
+            "task is to classify whether the response contains race-based, harmful, or "
+            "inaccurate content. This benchmark supports research into bias detection and "
+            "fairness in clinical AI systems.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Identify race-based bias in LLM-generated medical responses",
+                when="Any",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )