crfm-helm 0.5.7__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/scenarios/omni_math_scenario.py

@@ -1,6 +1,7 @@
 import datasets
 import os
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     Input,
     Output,
     CORRECT_TAG,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -51,3 +53,19 @@ class OmniMATHScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="Omni-MATH",
+            description=self.description,
+            main_metric="omni_math_accuracy",
+            main_split="test",
+            taxonomy=TaxonomyInfo(
+                task="mathematics",
+                what="universal Olympiad level mathematic benchmark",
+                who="human annotators",
+                when="2024",
+                language="English",
+            ),
+        )

helm/benchmark/scenarios/open_assistant_scenario.py

@@ -2,6 +2,7 @@ from typing import List, Dict, Any, DefaultDict
 from datasets import load_dataset, Dataset
 from collections import defaultdict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     TRAIN_SPLIT,
     VALID_SPLIT,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -126,3 +128,23 @@ class OpenAssistantScenario(Scenario):
         valid_instances = get_split_instances(dataset["validation"], VALID_SPLIT)
 
         return train_instances + valid_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="open_assistant",
+            display_name="Open Assistant",
+            short_display_name="Open Assistant",
+            description="LAION’s OpenAssistant Conversations Dataset (OASST1) that consists of 66,497 "
+            "conversation trees ([Köpf et al., "
+            "2023](https://openreview.net/forum?id=VSJotgbPHF)). We only use the initial "
+            "prompt in each conversation.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Human-written dialogues and response rankings",
+                when="2023",
+                who="Open Assistant participants",
+                language="35 languages",
+            ),
+            main_metric="Helpfulness",
+            main_split="valid",
+        )

helm/benchmark/scenarios/pubmed_qa_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -186,3 +188,23 @@ class PubMedQAScenario(Scenario):
                         instances.append(instance)
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="pubmed_qa",
+            display_name="PubMedQA",
+            description="PubMedQA is a biomedical question-answering dataset that evaluates a model's "
+            "ability to interpret scientific literature. It consists of PubMed abstracts "
+            "paired with yes/no/maybe questions derived from the content. The benchmark "
+            "assesses a model's capability to reason over biomedical texts and provide "
+            "factually grounded answers.",
+            taxonomy=TaxonomyInfo(
+                task="Question answering",
+                what="Answer questions based on PubMed abstracts",
+                when="Any",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/quac_scenario.py

@@ -3,6 +3,7 @@ import os
 import random
 from typing import List, Tuple
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -192,3 +194,15 @@ class QuACScenario(Scenario):
             instances.extend(self.get_split_instances(split_path, split=split_tag))
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="quac",
+            display_name="QuAC (Question Answering in Context)",
+            short_display_name="QuAC",
+            description="The QuAC benchmark for question answering in the context of dialogues [(Choi "
+            "et al., 2018)](https://aclanthology.org/D18-1241/).",
+            taxonomy=TaxonomyInfo(task="question answering", what="?", when="?", who="?", language="English"),
+            main_metric="f1_score",
+            main_split="valid",
+        )

helm/benchmark/scenarios/race_based_med_scenario.py

@@ -4,6 +4,7 @@ import os
 from typing import Dict, List
 from docx import Document
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_file_downloaded
 
@@ -150,3 +152,24 @@ class RaceBasedMedScenario(Scenario):
             )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="race_based_med",
+            display_name="RaceBias",
+            description="RaceBias is a benchmark used to evaluate language models for racially biased "
+            "or inappropriate content in medical question-answering scenarios. Each "
+            "instance consists of a medical question and a model-generated response. The "
+            "task is to classify whether the response contains race-based, harmful, or "
+            "inaccurate content. This benchmark supports research into bias detection and "
+            "fairness in clinical AI systems.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Identify race-based bias in LLM-generated medical responses",
+                when="Any",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/raft_scenario.py

@@ -5,6 +5,7 @@ import datasets
 from pathlib import Path
 from typing import List, Dict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -15,6 +16,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 PROMPT_SETTINGS_URL = "https://www.dropbox.com/s/a5cyevryzw8rt4f/prompt_construction_settings.json?dl=0"
@@ -144,3 +146,16 @@ class RAFTScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="raft",
+            display_name="RAFT (Real-world Annotated Few-Shot)",
+            short_display_name="RAFT",
+            description="The Real-world annotated few-shot (RAFT) meta-benchmark of 11 real-world text "
+            "classification tasks [(Alex et al., "
+            "2021)](https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/ca46c1b9512a7a8315fa3c5a946e8265-Abstract-round2.html).",
+            taxonomy=TaxonomyInfo(task="text classification", what="?", when="?", who="?", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/real_toxicity_prompts_scenario.py

@@ -3,8 +3,9 @@ import os
 import random
 from typing import List, Dict, Optional
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input
+from helm.benchmark.scenarios.scenario import Scenario, Instance, TEST_SPLIT, Input, ScenarioMetadata
 
 TOXIC_SUB_SPLIT: str = "toxic"
 NONTOXIC_SUB_SPLIT: str = "non-toxic"
@@ -57,3 +58,15 @@ class RealToxicityPromptsScenario(Scenario):
         random.shuffle(instances)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="real_toxicity_prompts",
+            display_name="RealToxicityPrompts",
+            description="The RealToxicityPrompts dataset for measuring toxicity in prompted model "
+            "generations [(Gehman et al., "
+            "2020)](https://aclanthology.org/2020.findings-emnlp.301/).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="unknown",
+            main_split="test",
+        )

helm/benchmark/scenarios/scenario.py

@@ -5,6 +5,7 @@ import os
 from pathlib import PurePath
 import inspect
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.media_object import MultimediaObject
 from helm.common.object_spec import ObjectSpec, create_object
 from helm.common.general import ensure_directory_exists, format_text, format_split, format_tags, indent_lines
@@ -189,6 +190,33 @@ class Instance:
         return info
 
 
+@dataclass(frozen=True)
+class ScenarioMetadata:
+    name: str
+    """Internal name (usually no spaces, etc.)"""
+
+    main_metric: str
+
+    main_split: str
+
+    display_name: Optional[str] = None
+    """What is displayed to the user"""
+
+    short_display_name: Optional[str] = None
+    """What is displayed to the user (e.g., in a table header)"""
+
+    description: Optional[str] = None
+    """Description of the scenario"""
+
+    short_description: Optional[str] = None
+    """Optional short description of the scenario.
+    This description is used in some space-constrained places in frontend tables.
+    If unset, the description field will be used instead."""
+
+    taxonomy: Optional[TaxonomyInfo] = None
+    """Optional taxonomy"""
+
+
 # TODO(#1212): Scenario should not be a dataclass.
 @dataclass
 class Scenario(ABC):
@@ -249,6 +277,9 @@ class Scenario(ABC):
             output.append("}")
         return output
 
+    def get_metadata(self) -> ScenarioMetadata:
+        raise NotImplementedError()
+
 
 def with_instance_ids(instances: List[Instance]) -> List[Instance]:
     """Return the instances with an ID.  Note: order of instances matters."""