crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/boolq_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import List, Dict, Optional
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     PassageQuestionInput,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -188,3 +190,21 @@ class BoolQScenario(Scenario):
             split_path: str = os.path.join(data_path, filename)
             instances.extend(self.get_split_instances(split, split_path, contrast_map if split == VALID_SPLIT else {}))
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="boolq",
+            display_name="BoolQ",
+            short_display_name=None,
+            description="The BoolQ benchmark for binary (yes/no) question answering [(Clark et al., "
+            "2019)](https://aclanthology.org/N19-1300/).",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="passages from Wikipedia, questions from search queries",
+                when="2010s",
+                who="web users",
+                language="English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="valid",
+        )

helm/benchmark/scenarios/chw_care_plan_scenario.py

@@ -1,6 +1,7 @@
 import pandas as pd
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
@@ -10,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -104,3 +106,24 @@ class CHWCarePlanScenario(Scenario):
             )
             instances.append(instance)
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="chw_care_plan",
+            display_name="NoteExtract",
+            description="NoteExtract is a benchmark that focuses on the structured extraction of "
+            "information from free-form clinical text. It provides care plan notes authored "
+            "by health workers and evaluates a model's ability to convert them into a "
+            "predefined structured format, such as fields for Chief Complaint and History "
+            "of Present Illness. The benchmark emphasizes faithful extraction without "
+            "hallucination or inference.",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Convert general text care plans into structured formats",
+                when="Any",
+                who="Clinician, Researcher",
+                language="English",
+            ),
+            main_metric="chw_care_plan_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/civil_comments_scenario.py

@@ -3,6 +3,7 @@ from typing import Dict, List, Set
 
 import pandas as pd
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Scenario,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -126,3 +128,14 @@ class CivilCommentsScenario(Scenario):
             )
             instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="civil_comments",
+            display_name="CivilComments",
+            description="The CivilComments benchmark for toxicity detection [(Borkan et al., "
+            "2019)](https://arxiv.org/pdf/1903.04561.pdf).",
+            taxonomy=TaxonomyInfo(task="toxicity classification", what="?", when="?", who="?", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/clear_scenario.py

@@ -2,6 +2,7 @@ import os
 import pandas as pd
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -155,3 +157,24 @@ class CLEARScenario(Scenario):
             )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="clear",
+            display_name="CLEAR",
+            description="CLEAR is a benchmark designed to evaluate models on their ability to detect "
+            "medical conditions from patient notes using categorical responses. Each "
+            "instance consists of a clinical note and a target condition, requiring the "
+            "model to classify the patient's history as either affirmative, negative, or "
+            "uncertain [(Lopez et al., "
+            "2025)](https://www.nature.com/articles/s41746-024-01377-1).",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Classify medical condition presence from patient notes",
+                when="Any",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )