crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/scenarios/mental_health_scenario.py

@@ -1,5 +1,6 @@
 import pandas as pd
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -8,6 +9,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import check_file_exists
 
@@ -121,3 +123,24 @@ class MentalHealthScenario(Scenario):
         instances = self.process_dialogue_data(dialogue_data)
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mental_health",
+            display_name="MentalHealth",
+            description="MentalHealth is a benchmark focused on evaluating empathetic communication in "
+            "mental health counseling. It includes real or simulated conversations between "
+            "patients and counselors, where the task is to generate compassionate and "
+            "appropriate counselor responses. The benchmark assesses a model's ability to "
+            "support patients emotionally and meaningfully engage in therapeutic "
+            "conversations.",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate empathetic counseling responses in mental health " "conversations",
+                when="Any",
+                who="Counselors, Patients",
+                language="English",
+            ),
+            main_metric="mental_health_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/mimic_bhc_scenario.py

@@ -1,6 +1,7 @@
 import json
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
@@ -10,11 +11,12 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 
 
 class MIMICBHCScenario(Scenario):
-    """
+    r"""
     MIMIC-IV-BHC presents a curated collection of preprocessed discharge notes with labeled brief hospital
     course (BHC) summaries. This dataset is derived from MIMIC-IV (https://doi.org/10.1093/jamia/ocae312).
 
@@ -101,3 +103,25 @@ class MIMICBHCScenario(Scenario):
                 )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mimic_bhc",
+            display_name="MIMIC-IV-BHC",
+            short_display_name="MIMIC-BHC",
+            description="MIMIC-BHC is a benchmark focused on summarization of discharge notes into "
+            "Brief Hospital Course (BHC) sections. It consists of curated discharge notes "
+            "from MIMIC-IV, each paired with its corresponding BHC summary. The benchmark "
+            "evaluates a model's ability to condense detailed clinical information into "
+            "accurate, concise summaries that reflect the patient's hospital stay [(Aali et "
+            "al., 2024)](https://doi.org/10.1093/jamia/ocae312).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Summarize the clinical note into a brief hospital course",
+                when="Upon hospital discharge",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="mimic_bhc_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/mimic_rrs_scenario.py

@@ -1,6 +1,7 @@
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
@@ -10,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -96,3 +98,24 @@ class MIMICRRSScenario(Scenario):
             lines = file.readlines()
         lines = [line.strip() for line in lines]
         return lines
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mimic_rrs",
+            display_name="MIMIC-RRS",
+            short_display_name="MIMIC-RRS",
+            description="MIMIC-RRS is a benchmark constructed from radiology reports in the MIMIC-III "
+            "database. It contains pairs of ‘Findings‘ and ‘Impression‘ sections, enabling "
+            "evaluation of a model's ability to summarize diagnostic imaging observations "
+            "into concise, clinically relevant conclusions [(Chen et al., "
+            "2023)](https://arxiv.org/abs/2211.08584).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate radiology report summaries from findings sections",
+                when="Post-imaging",
+                who="Radiologist",
+                language="English",
+            ),
+            main_metric="mimic_rrs_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/mimiciv_billing_code_scenario.py

@@ -2,6 +2,7 @@ import pandas as pd
 import numpy as np
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -75,3 +77,23 @@ class MIMICIVBillingCodeScenario(Scenario):
                 continue
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mimiciv_billing_code",
+            display_name="MIMIC-IV Billing Code",
+            description="MIMIC-IV Billing Code is a benchmark derived from discharge summaries in the "
+            "MIMIC-IV database, paired with their corresponding ICD-10 billing codes. The "
+            "task requires models to extract structured billing codes based on free-text "
+            "clinical notes, reflecting real-world hospital coding tasks for financial "
+            "reimbursement.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Predict ICD-10 billing codes from clinical discharge notes",
+                when="During or after patient discharge",
+                who="Hospital Admistrator",
+                language="English",
+            ),
+            main_metric="mimiciv_billing_code_f1",
+            main_split="test",
+        )

helm/benchmark/scenarios/mmlu_pro_scenario.py

@@ -2,6 +2,7 @@ from typing import Dict, List
 from datasets import Dataset, load_dataset
 
 from helm.common.hierarchical_logger import hlog
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -93,3 +95,19 @@ class MMLUProScenario(Scenario):
             instances.extend(self.process_dataset(data, split))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="MMLU-Pro",
+            description=self.description,
+            main_metric="chain_of_thought_correctness",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="graduate-level questions in biology, physics, and chemistry",
+                who="domain experts",
+                when="2023",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/mmlu_scenario.py

@@ -2,6 +2,7 @@ import csv
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -124,3 +126,16 @@ class MMLUScenario(Scenario):
             instances.extend(self.process_csv(csv_path, splits[split]))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="mmlu",
+            display_name="MMLU (Massive Multitask Language Understanding)",
+            short_display_name="MMLU",
+            description="The Massive Multitask Language Understanding (MMLU) benchmark for "
+            "knowledge-intensive question answering across 57 domains [(Hendrycks et al., "
+            "2021)](https://openreview.net/forum?id=d7KBjmI3GmQ).",
+            taxonomy=TaxonomyInfo(task="question answering", what="?", when="?", who="?", language="English"),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/mmmlu_scenario.py

@@ -0,0 +1,85 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MMMLUScenario(Scenario):
+    """Multilingual Massive Multitask Language Understanding (MMMLU) by OpenAI
+
+    The MMLU is a widely recognized benchmark of general knowledge attained
+    by AI models. It covers a broad range of topics from 57 different categories,
+    covering elementary-level knowledge up to advanced professional subjects like
+    law, physics, history, and computer science.
+
+    MMMLU is a translation of MMLU’s test set into 14 languages using professional
+    human translators. Relying on human translators for this evaluation increases
+    confidence in the accuracy of the translations, especially for low-resource
+    languages like Yoruba.
+
+    The Massive Multitask Language Understanding benchmark from this paper:
+
+    - https://arxiv.org/pdf/2009.03300.pdf
+
+    The MMMLU dataset is from here:
+
+    - https://huggingface.co/datasets/openai/MMMLU
+    """
+
+    name = "mmmlu"
+    description = "Multilingual Massive Multitask Language Understanding"
+    tags = ["knowledge", "multiple_choice"]
+
+    OPTIONS = ["A", "B", "C", "D"]
+
+    def __init__(self, locale: str, subject: str):
+        super().__init__()
+        self.locale: str = locale
+        self.subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "openai/MMMLU",
+            self.locale,
+            revision="325a01dc3e173cac1578df94120499aaca2e2504",
+            cache_dir=cache_dir,
+            split="test",
+        )
+        assert isinstance(dataset, datasets.Dataset)
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row_index, row in enumerate(dataset):
+            if self.subject != "all" and row["Subject"] != self.subject:
+                continue
+            input = Input(text=row["Question"])
+            references: List[Reference] = []
+            for option in self.OPTIONS:
+                references.append(
+                    Reference(
+                        output=Output(text=row[option]),
+                        tags=[CORRECT_TAG] if option == row["Answer"] else [],
+                    )
+                )
+            instance = Instance(
+                id=f"id{row_index}",
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/msmarco_scenario.py

@@ -4,6 +4,7 @@ import random
 from collections import defaultdict
 from typing import Dict, List, Tuple, Optional, Union
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     TRAIN_SPLIT,
     VALID_SPLIT,
     CORRECT_TAG,
+    ScenarioMetadata,
     make_rank_tag,
     make_relevance_tag,
     Input,
@@ -657,3 +659,31 @@ class MSMARCOScenario(Scenario):
         valid_instances = self.get_valid_instances()
 
         return train_instances + valid_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.track == self.REGULAR_TRACK:
+            return ScenarioMetadata(
+                name="msmarco_regular",
+                display_name="MS MARCO (regular track)",
+                short_display_name="MS MARCO (regular)",
+                description="The MS MARCO benchmark's regular track for passage retrieval in information "
+                "retrieval "
+                "[(https://microsoft.github.io/msmarco/)](https://microsoft.github.io/msmarco/).",
+                taxonomy=TaxonomyInfo(task="information retrieval", what="?", when="?", who="?", language="English"),
+                main_metric="RR@10",
+                main_split="valid",
+            )
+        elif self.track == self.TREC_TRACK:
+            return ScenarioMetadata(
+                name="msmarco_trec",
+                display_name="MS MARCO (TREC track)",
+                short_display_name="MS MARCO (TREC)",
+                description="The MS MARCO benchmark's deep learning TREC track for passage retrieval in "
+                "information retrieval "
+                "[(https://trec.nist.gov)](https://microsoft.github.io/msmarco/).",
+                taxonomy=TaxonomyInfo(task="information retrieval", what="?", when="?", who="?", language="English"),
+                main_metric="NDCG@10",
+                main_split="valid",
+            )
+        else:
+            raise Exception(f"Unknown track {self.track}")

helm/benchmark/scenarios/mtsamples_procedures_scenario.py

@@ -1,6 +1,7 @@
 import os
 import requests
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -142,3 +144,23 @@ class MTSamplesProceduresScenario(Scenario):
                 print(f"Error processing {file_name}: {e}")
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mtsamples_procedures",
+            display_name="MTSamples Procedures",
+            description="MTSamples Procedures is a benchmark composed of transcribed operative notes, "
+            "focused on documenting surgical procedures. Each example presents a brief "
+            "patient case involving a surgical intervention, and the model is tasked with "
+            "generating a coherent and clinically accurate procedural summary or treatment "
+            "plan.",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Document and extract information about medical procedures",
+                when="Post-procedure",
+                who="Clinician, Researcher",
+                language="English",
+            ),
+            main_metric="mtsamples_procedures_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/mtsamples_replicate_scenario.py

@@ -1,6 +1,7 @@
 import os
 import requests
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -140,3 +142,23 @@ class MTSamplesReplicateScenario(Scenario):
                 print(f"Error processing {file_name}: {e}")
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="mtsamples_replicate",
+            display_name="MTSamples",
+            short_display_name="MTSamples",
+            description="MTSamples Replicate is a benchmark that provides transcribed medical reports "
+            "from various specialties. It is used to evaluate a model's ability to generate "
+            "clinically appropriate treatment plans based on unstructured patient "
+            "documentation [(MTSamples, 2025)](https://mtsamples.com).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Generate treatment plans based on clinical notes",
+                when="Post-diagnosis",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="mtsamples_replicate_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/n2c2_ct_matching_scenario.py

@@ -4,6 +4,7 @@ import re
 from typing import Any, Dict, List, Tuple, Optional
 import xml.etree.ElementTree as ET
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 
 ORIGINAL_DEFINITIONS = {
@@ -275,3 +277,21 @@ class N2C2CTMatchingScenario(Scenario):
                     )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="n2c2_ct_matching",
+            display_name="N2C2-CT Matching",
+            short_display_name="N2C2-CT",
+            description="A dataset that provides clinical notes and asks the model to classify whether "
+            "the patient is a valid candidate for a provided clinical trial.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Classify whether a patient is a valid candidate for a clinical " "trial based on clinical notes",
+                when="Pre-Trial",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/narrativeqa_scenario.py

@@ -3,6 +3,7 @@ import random
 import csv
 from typing import List, Dict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     Input,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -155,3 +157,21 @@ class NarrativeQAScenario(Scenario):
             instances.extend(self.get_split_instances(summaries_file=summaries_file, qaps_file=qaps_file, split=split))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="narrative_qa",
+            display_name="NarrativeQA",
+            short_display_name=None,
+            description="The NarrativeQA benchmark for reading comprehension over narratives [(Kočiský "
+            "et al., 2017)](https://aclanthology.org/Q18-1023/).",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="passages are books and movie scripts, questions are unknown",
+                when="?",
+                who="?",
+                language="English",
+            ),
+            main_metric="f1_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/natural_qa_scenario.py

@@ -6,6 +6,7 @@ import os
 import re
 import html
 import random
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import htrack_block, hlog
 from typing import List, Dict
 
@@ -20,6 +21,7 @@ from helm.benchmark.scenarios.scenario import (
     PassageQuestionInput,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -324,3 +326,33 @@ class NaturalQAScenario(Scenario):
             instances.extend(self.get_file_instances(target_path))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.context_mode == "closedbook":
+            name = "natural_qa_closedbook"
+            display_name = "NaturalQuestions (closed-book)"
+        elif self.context_mode == "openbook_longans":
+            name = "natural_qa_openbook_longans"
+            display_name = "NaturalQuestions (open-book)"
+        elif self.context_mode == "openbook_wiki":
+            name = "natural_qa_openbook_wiki"
+            display_name = "NaturalQuestions (open-book Wiki)"
+        else:
+            raise Exception(f"Unknown context_mode {self.context_mode}")
+        return ScenarioMetadata(
+            name=name,
+            display_name=display_name,
+            description="The NaturalQuestions [(Kwiatkowski et al., "
+            "2019)](https://aclanthology.org/Q19-1026/) benchmark for question answering "
+            "based on naturally-occurring queries through Google Search. The input does not "
+            "include the Wikipedia page with the answer.",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="passages from Wikipedia, questions from search queries",
+                when="2010s",
+                who="web users",
+                language="English",
+            ),
+            main_metric="f1_score",
+            main_split="valid",
+        )

helm/benchmark/scenarios/omni_math_scenario.py

@@ -1,6 +1,7 @@
 import datasets
 import os
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     Input,
     Output,
     CORRECT_TAG,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -51,3 +53,19 @@ class OmniMATHScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="Omni-MATH",
+            description=self.description,
+            main_metric="omni_math_accuracy",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="mathematics",
+                what="universal Olympiad level mathematic benchmark",
+                who="human annotators",
+                when="2024",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/open_assistant_scenario.py

@@ -2,6 +2,7 @@ from typing import List, Dict, Any, DefaultDict
 from datasets import load_dataset, Dataset
 from collections import defaultdict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     TRAIN_SPLIT,
     VALID_SPLIT,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -126,3 +128,23 @@ class OpenAssistantScenario(Scenario):
         valid_instances = get_split_instances(dataset["validation"], VALID_SPLIT)
 
         return train_instances + valid_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="open_assistant",
+            display_name="Open Assistant",
+            short_display_name="Open Assistant",
+            description="LAION’s OpenAssistant Conversations Dataset (OASST1) that consists of 66,497 "
+            "conversation trees ([Köpf et al., "
+            "2023](https://openreview.net/forum?id=VSJotgbPHF)). We only use the initial "
+            "prompt in each conversation.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Human-written dialogues and response rankings",
+                when="2023",
+                who="Open Assistant participants",
+                language="35 languages",
+            ),
+            main_metric="Helpfulness",
+            main_split="valid",
+        )