crfm-helm 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

helm/benchmark/scenarios/mmlu_scenario.py

@@ -2,6 +2,7 @@ import csv
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -124,3 +126,22 @@ class MMLUScenario(Scenario):
             instances.extend(self.process_csv(csv_path, splits[split]))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="mmlu",
+            display_name="MMLU (Massive Multitask Language Understanding)",
+            short_display_name="MMLU",
+            description="The Massive Multitask Language Understanding (MMLU) benchmark for "
+            "knowledge-intensive question answering across 57 domains [(Hendrycks et al., "
+            "2021)](https://arxiv.org/pdf/2009.03300.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="math, science, history, etc.",
+                when="before 2021",
+                who="various online sources",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/msmarco_scenario.py

@@ -4,6 +4,7 @@ import random
 from collections import defaultdict
 from typing import Dict, List, Tuple, Optional, Union
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     TRAIN_SPLIT,
     VALID_SPLIT,
     CORRECT_TAG,
+    ScenarioMetadata,
     make_rank_tag,
     make_relevance_tag,
     Input,
@@ -657,3 +659,31 @@ class MSMARCOScenario(Scenario):
         valid_instances = self.get_valid_instances()
 
         return train_instances + valid_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.track == self.REGULAR_TRACK:
+            return ScenarioMetadata(
+                name="msmarco_regular",
+                display_name="MS MARCO (regular track)",
+                short_display_name="MS MARCO (regular)",
+                description="The MS MARCO benchmark's regular track for passage retrieval in information "
+                "retrieval "
+                "[(https://microsoft.github.io/msmarco/)](https://microsoft.github.io/msmarco/).",
+                taxonomy=TaxonomyInfo(task="information retrieval", what="?", when="?", who="?", language="English"),
+                main_metric="RR@10",
+                main_split="valid",
+            )
+        elif self.track == self.TREC_TRACK:
+            return ScenarioMetadata(
+                name="msmarco_trec",
+                display_name="MS MARCO (TREC track)",
+                short_display_name="MS MARCO (TREC)",
+                description="The MS MARCO benchmark's deep learning TREC track for passage retrieval in "
+                "information retrieval "
+                "[(https://trec.nist.gov)](https://microsoft.github.io/msmarco/).",
+                taxonomy=TaxonomyInfo(task="information retrieval", what="?", when="?", who="?", language="English"),
+                main_metric="NDCG@10",
+                main_split="valid",
+            )
+        else:
+            raise Exception(f"Unknown track {self.track}")

helm/benchmark/scenarios/mtsamples_procedures_scenario.py

@@ -1,6 +1,7 @@
 import os
 import requests
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -142,3 +144,23 @@ class MTSamplesProceduresScenario(Scenario):
                 print(f"Error processing {file_name}: {e}")
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mtsamples_procedures",
+            display_name="MTSamples Procedures",
+            description="MTSamples Procedures is a benchmark composed of transcribed operative notes, "
+            "focused on documenting surgical procedures. Each example presents a brief "
+            "patient case involving a surgical intervention, and the model is tasked with "
+            "generating a coherent and clinically accurate procedural summary or treatment "
+            "plan.",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Document and extract information about medical procedures",
+                when="Post-procedure",
+                who="Clinician, Researcher",
+                language="English",
+            ),
+            main_metric="mtsamples_procedures_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/mtsamples_replicate_scenario.py

@@ -1,6 +1,7 @@
 import os
 import requests
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -140,3 +142,23 @@ class MTSamplesReplicateScenario(Scenario):
                 print(f"Error processing {file_name}: {e}")
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mtsamples_replicate",
+            display_name="MTSamples",
+            short_display_name="MTSamples",
+            description="MTSamples Replicate is a benchmark that provides transcribed medical reports "
+            "from various specialties. It is used to evaluate a model's ability to generate "
+            "clinically appropriate treatment plans based on unstructured patient "
+            "documentation [(MTSamples, 2025)](https://mtsamples.com).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate treatment plans based on clinical notes",
+                when="Post-diagnosis",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="mtsamples_replicate_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/n2c2_ct_matching_scenario.py

@@ -4,6 +4,7 @@ import re
 from typing import Any, Dict, List, Tuple, Optional
 import xml.etree.ElementTree as ET
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 
 ORIGINAL_DEFINITIONS = {
@@ -275,3 +277,21 @@ class N2C2CTMatchingScenario(Scenario):
                     )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="n2c2_ct_matching",
+            display_name="N2C2-CT Matching",
+            short_display_name="N2C2-CT",
+            description="A dataset that provides clinical notes and asks the model to classify whether "
+            "the patient is a valid candidate for a provided clinical trial.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Classify whether a patient is a valid candidate for a clinical " "trial based on clinical notes",
+                when="Pre-Trial",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/narrativeqa_scenario.py

@@ -3,6 +3,7 @@ import random
 import csv
 from typing import List, Dict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     Input,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -155,3 +157,20 @@ class NarrativeQAScenario(Scenario):
             instances.extend(self.get_split_instances(summaries_file=summaries_file, qaps_file=qaps_file, split=split))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="narrative_qa",
+            display_name="NarrativeQA",
+            description="The NarrativeQA benchmark for reading comprehension over narratives [(Kočiský "
+            "et al., 2017)](https://aclanthology.org/Q18-1023/).",
+            taxonomy=TaxonomyInfo(
+                task="short-answer question answering",
+                what="passages are books and movie scripts, questions are unknown",
+                when="2018",
+                who="annotators from summaries",
+                language="English",
+            ),
+            main_metric="f1_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/natural_qa_scenario.py

@@ -6,6 +6,7 @@ import os
 import re
 import html
 import random
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import htrack_block, hlog
 from typing import List, Dict
 
@@ -20,6 +21,7 @@ from helm.benchmark.scenarios.scenario import (
     PassageQuestionInput,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -324,3 +326,33 @@ class NaturalQAScenario(Scenario):
             instances.extend(self.get_file_instances(target_path))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.context_mode == "closedbook":
+            name = "natural_qa_closedbook"
+            display_name = "NaturalQuestions (closed-book)"
+        elif self.context_mode == "openbook_longans":
+            name = "natural_qa_openbook_longans"
+            display_name = "NaturalQuestions (open-book)"
+        elif self.context_mode == "openbook_wiki":
+            name = "natural_qa_openbook_wiki"
+            display_name = "NaturalQuestions (open-book Wiki)"
+        else:
+            raise Exception(f"Unknown context_mode {self.context_mode}")
+        return ScenarioMetadata(
+            name=name,
+            display_name=display_name,
+            description="The NaturalQuestions [(Kwiatkowski et al., "
+            "2019)](https://aclanthology.org/Q19-1026/) benchmark for question answering "
+            "based on naturally-occurring queries through Google Search. The input does not "
+            "include the Wikipedia page with the answer.",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="passages from Wikipedia, questions from search queries",
+                when="2010s",
+                who="web users",
+                language="English",
+            ),
+            main_metric="f1_score",
+            main_split="valid",
+        )

helm/benchmark/scenarios/omni_math_scenario.py

@@ -1,6 +1,7 @@
 import datasets
 import os
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     Input,
     Output,
     CORRECT_TAG,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -51,3 +53,19 @@ class OmniMATHScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="Omni-MATH",
+            description=self.description,
+            main_metric="omni_math_accuracy",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="mathematics",
+                what="universal Olympiad level mathematic benchmark",
+                who="human annotators",
+                when="2024",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/open_assistant_scenario.py

@@ -2,6 +2,7 @@ from typing import List, Dict, Any, DefaultDict
 from datasets import load_dataset, Dataset
 from collections import defaultdict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     TRAIN_SPLIT,
     VALID_SPLIT,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -126,3 +128,23 @@ class OpenAssistantScenario(Scenario):
         valid_instances = get_split_instances(dataset["validation"], VALID_SPLIT)
 
         return train_instances + valid_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="open_assistant",
+            display_name="Open Assistant",
+            short_display_name="Open Assistant",
+            description="LAION’s OpenAssistant Conversations Dataset (OASST1) that consists of 66,497 "
+            "conversation trees ([Köpf et al., "
+            "2023](https://openreview.net/forum?id=VSJotgbPHF)). We only use the initial "
+            "prompt in each conversation.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Human-written dialogues and response rankings",
+                when="2023",
+                who="Open Assistant participants",
+                language="35 languages",
+            ),
+            main_metric="Helpfulness",
+            main_split="valid",
+        )

helm/benchmark/scenarios/openai_mrcr_scenario.py

@@ -6,6 +6,7 @@ from typing import List, Optional
 import datasets
 import tiktoken
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Output,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     Instance,
     TEST_SPLIT,
     Input,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -77,3 +79,16 @@ class OpenAIMRCRScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="openai_mrcr",
+            display_name="OpenAI MRCR",
+            description="OpenAI MRCR (Multi-round co-reference resolution) is a long context dataset "
+            "for benchmarking an LLM's ability to distinguish between multiple needles "
+            "hidden in context. This eval is inspired by the MRCR eval first introduced by "
+            "[Vodrahalli et al., 2024](https://arxiv.org/pdf/2409.12640v2).",
+            taxonomy=TaxonomyInfo(task="MRCR", what="Synthetic data", when="2025", who="None", language="English"),
+            main_metric="openai_mrcr_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/pubmed_qa_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -186,3 +188,23 @@ class PubMedQAScenario(Scenario):
                         instances.append(instance)
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="pubmed_qa",
+            display_name="PubMedQA",
+            description="PubMedQA is a biomedical question-answering dataset that evaluates a model's "
+            "ability to interpret scientific literature. It consists of PubMed abstracts "
+            "paired with yes/no/maybe questions derived from the content. The benchmark "
+            "assesses a model's capability to reason over biomedical texts and provide "
+            "factually grounded answers.",
+            taxonomy=TaxonomyInfo(
+                task="Question answering",
+                what="Answer questions based on PubMed abstracts",
+                when="Any",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/quac_scenario.py

@@ -3,6 +3,7 @@ import os
 import random
 from typing import List, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -192,3 +194,15 @@ class QuACScenario(Scenario):
             instances.extend(self.get_split_instances(split_path, split=split_tag))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="quac",
+            display_name="QuAC (Question Answering in Context)",
+            short_display_name="QuAC",
+            description="The QuAC benchmark for question answering in the context of dialogues [(Choi "
+            "et al., 2018)](https://aclanthology.org/D18-1241/).",
+            taxonomy=TaxonomyInfo(task="question answering", what="?", when="?", who="?", language="English"),
+            main_metric="f1_score",
+            main_split="valid",
+        )

helm/benchmark/scenarios/race_based_med_scenario.py

@@ -4,6 +4,7 @@ import os
 from typing import Dict, List
 from docx import Document
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_file_downloaded
 
@@ -150,3 +152,24 @@ class RaceBasedMedScenario(Scenario):
             )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="race_based_med",
+            display_name="RaceBias",
+            description="RaceBias is a benchmark used to evaluate language models for racially biased "
+            "or inappropriate content in medical question-answering scenarios. Each "
+            "instance consists of a medical question and a model-generated response. The "
+            "task is to classify whether the response contains race-based, harmful, or "
+            "inaccurate content. This benchmark supports research into bias detection and "
+            "fairness in clinical AI systems.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Identify race-based bias in LLM-generated medical responses",
+                when="Any",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/raft_scenario.py

@@ -5,6 +5,7 @@ import datasets
 from pathlib import Path
 from typing import List, Dict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -15,6 +16,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 PROMPT_SETTINGS_URL = "https://www.dropbox.com/s/a5cyevryzw8rt4f/prompt_construction_settings.json?dl=0"
@@ -144,3 +146,16 @@ class RAFTScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="raft",
+            display_name="RAFT (Real-world Annotated Few-Shot)",
+            short_display_name="RAFT",
+            description="The Real-world annotated few-shot (RAFT) meta-benchmark of 11 real-world text "
+            "classification tasks [(Alex et al., "
+            "2021)](https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/ca46c1b9512a7a8315fa3c5a946e8265-Abstract-round2.html).",
+            taxonomy=TaxonomyInfo(task="text classification", what="?", when="?", who="?", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/real_toxicity_prompts_scenario.py

@@ -3,8 +3,9 @@ import os
 import random
 from typing import List, Dict, Optional
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 TOXIC_SUB_SPLIT: str = "toxic"
 NONTOXIC_SUB_SPLIT: str = "non-toxic"
@@ -57,3 +58,15 @@ class RealToxicityPromptsScenario(Scenario):
         random.shuffle(instances)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="real_toxicity_prompts",
+            display_name="RealToxicityPrompts",
+            description="The RealToxicityPrompts dataset for measuring toxicity in prompted model "
+            "generations [(Gehman et al., "
+            "2020)](https://aclanthology.org/2020.findings-emnlp.301/).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="unknown",
+            main_split="test",
+        )

helm/benchmark/scenarios/ruler_qa_scenarios.py

@@ -1,6 +1,7 @@
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
 from helm.benchmark.scenarios.ruler_qa_scenario_helper import generate_samples  # type: ignore
 from helm.benchmark.scenarios.scenario import (
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -78,6 +80,25 @@ class RULERHotpotQAScenario(_RULERQAScenario):
     def __init__(self, max_num_words: int):
         super().__init__("hotpotqa", max_num_words)
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="ruler_hotpotqa",
+            display_name="RULER HotPotQA",
+            description="RULER HotPotQA is an augmented version of HotPotQA ([Yang et al., "
+            "2018](https://arxiv.org/abs/1809.09600)) introduced by [Hsieh et al., "
+            "2024](https://arxiv.org/abs/2404.06654) to simulate a multi-hop question "
+            "answering as a long-context scenario.",
+            taxonomy=TaxonomyInfo(
+                task="question answering with retrieval-augmented generation",
+                what="Wikipedia articles",
+                when="Before 2018",
+                who="Wikipedia authors",
+                language="English",
+            ),
+            main_metric="ruler_string_match_part",
+            main_split="valid",
+        )
+
 
 class RULERSQuADScenario(_RULERQAScenario):
     name = "ruler_squad"
@@ -86,3 +107,22 @@ class RULERSQuADScenario(_RULERQAScenario):
 
     def __init__(self, max_num_words: int):
         super().__init__("squad", max_num_words)
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="ruler_squad",
+            display_name="RULER SQuAD",
+            description="RULER SQuAD is an augmented version of SQuAD ([Rajpurkar et al., "
+            "2018](https://arxiv.org/abs/1806.03822)) introduced by [Hsieh et al., "
+            "2024](https://arxiv.org/abs/2404.06654) to simulate a single-hop question "
+            "answering as a long-context scenario.",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="Wikipedia articles",
+                when="Before 2018",
+                who="Wikipedia authors and crowdworkers",
+                language="English",
+            ),
+            main_metric="ruler_string_match_part",
+            main_split="valid",
+        )