crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/scenarios/civil_comments_scenario.py

@@ -3,6 +3,7 @@ from typing import Dict, List, Set
 
 import pandas as pd
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Scenario,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -126,3 +128,14 @@ class CivilCommentsScenario(Scenario):
             )
             instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="civil_comments",
+            display_name="CivilComments",
+            description="The CivilComments benchmark for toxicity detection [(Borkan et al., "
+            "2019)](https://arxiv.org/pdf/1903.04561.pdf).",
+            taxonomy=TaxonomyInfo(task="toxicity classification", what="?", when="?", who="?", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/clear_scenario.py

@@ -2,6 +2,7 @@ import os
 import pandas as pd
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Reference,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -155,3 +157,24 @@ class CLEARScenario(Scenario):
             )
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="clear",
+            display_name="CLEAR",
+            description="CLEAR is a benchmark designed to evaluate models on their ability to detect "
+            "medical conditions from patient notes using categorical responses. Each "
+            "instance consists of a clinical note and a target condition, requiring the "
+            "model to classify the patient's history as either affirmative, negative, or "
+            "uncertain [(Lopez et al., "
+            "2025)](https://www.nature.com/articles/s41746-024-01377-1).",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Classify medical condition presence from patient notes",
+                when="Any",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )