crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/scenarios/codeinsights_student_mistake_scenario.py

@@ -0,0 +1,188 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsStudentMistakeScenario(Scenario):
+    name = "codeinsights_student_mistake"
+    description = "Mimic how students mistake their C++ codes on foundational questions"
+    tags = ["codeinsights", "c++", "student_mistake"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario3_data.csv")
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        instances = []
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values(by=["student_id", "question_unittest_id", "timestamp"])
+            if len(student_df) < 4:
+                continue
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Get test cases for this question
+            question_id = target.get("question_unittest_id", None)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass (0 or 1) for the target question
+            student_correctness_pattern = target.get("pass", None)
+            main_part = int(student_correctness_pattern)  # "1111111111"
+            # Convert each character to an int
+            student_correctness_list = [int(ch) for ch in str(main_part)]  # [1,1,1,1,1,1,1,1,1,1]
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            prompt = (
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                "When students submit a code to the platform, it will be tested by number of unit tests, where"
+                "- Unit test pass rate = proportion of unit tests passed with the code \n"
+                "- Full pass rate   = proportion of code passing all unit tests\n\n"
+                "=== Past Mistake Examples ===\n"
+                "Example 1 (Week {first['week']}, Topic: {first['topic']}):\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{first['response_mistake']}\n\n"
+                "Example 2 (Week {second['week']}, Topic: {second['topic']}):\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{second['response_mistake']}\n\n"
+                "Example 3 (Week {third['week']}, Topic: {third['topic']}):\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{third['response_mistake']}\n\n"
+                "=== New Target Problem ===\n"
+                f"Week: {target['week']}, Topic: {target['topic']}\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "⚠**Instructions:**\n"
+                "1. Mimic your own coding style, naming conventions, indentation, and typical error patterns.\n"
+                "2. Introduce mistake you are likely to make (e.g., off‐by‐one index, wrong initialization, "
+                "missing edge case).\n"
+                "3. Do **not** produce a fully correct solution or add unfamiliar optimizations.\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code is includes any class definition when needed. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+
+            print(f"\n=== DEBUG INFO FOR STUDENT {student_id}, QUESTION {question_id} ===")
+            print(f"Test cases loaded: {len(question_test_cases)}")
+            print(f"Student correctness pattern: {student_correctness_list}")
+            print(f"Original pass field: {target.get('pass', 'MISSING')}")
+            print(f"Question template exists: {'question_template' in target}")
+            print(f"Question name: {target.get('question_name', 'MISSING')}")
+
+            # Also add this validation in your UnitTestAlignmentMetric evaluate_generation method:
+            def evaluate_generation(self, adapter_spec, request_state, metric_service, eval_cache_path):
+                print("\n=== UNIT TEST METRIC DEBUG ===")
+                print(f"Has extra_data: {hasattr(request_state.instance, 'extra_data')}")
+                if hasattr(request_state.instance, "extra_data"):
+                    extra_data = request_state.instance.extra_data
+                    print(f"Extra data keys: {list(extra_data.keys())}")
+                    print(f"Test cases: {len(extra_data.get('test_cases', []))}")
+                    print(f"Student pattern: {extra_data.get('student_correctness_pattern', 'MISSING')}")
+
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response_mistake"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/commonsense_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
@@ -14,6 +15,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -72,6 +74,19 @@ class HellaSwagScenario(Scenario):
         assert len(answers) == 4
         return _make_instance(question=question, answers=answers, correct_answer=correct_answer, split=split)
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="hellaswag",
+            display_name="HellaSwag",
+            description="The HellaSwag benchmark for commonsense reasoning in question answering "
+            "[(Zellers et al., 2019)](https://aclanthology.org/P19-1472/).",
+            taxonomy=TaxonomyInfo(
+                task="question answering", what="commonsense reasoning", when="?", who="?", language="English"
+            ),
+            main_metric="exact_match",
+            main_split="valid",
+        )
+
 
 class OpenBookQA(Scenario):
     name = "openbookqa"
@@ -113,6 +128,17 @@ class OpenBookQA(Scenario):
         assert item["question"]["choices"][correct_choice]["label"] == item["answerKey"]
         return _make_instance(question=question, answers=answers, correct_answer=correct_answer, split=split)
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="openbookqa",
+            display_name="OpenbookQA",
+            description="The OpenbookQA benchmark for commonsense-intensive open book question "
+            "answering [(Mihaylov et al., 2018)](https://aclanthology.org/D18-1260/).",
+            taxonomy=TaxonomyInfo(task="question answering", what="?", when="?", who="?", language="English"),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CommonSenseQAScenario(Scenario):
     name = "commonsenseqa"

helm/benchmark/scenarios/compositional_instructions.yaml

@@ -0,0 +1,70 @@
+rules:
+- category: Root
+  expansions:
+    - text: ${Task} ${Style}
+
+- category: Task
+  expansions:
+    - text: Explain ${HowTo}
+    - text: Explain ${Topic}
+    - text: Tell me a joke about ${Topic}
+    - text: Tell me a joke about ${TopicPair}
+    - text: What are the similarities between ${TopicPair}
+    - text: What are the differences between ${TopicPair}
+    - text: Tell me 5 surprising facts about ${Topic}
+    - text: Persuade me to learn about ${Topic}
+
+- category: HowTo
+  expansions:
+    - text: how airplanes fly
+    - text: how maglev trains work
+    - text: how to grow tomatoes in the wintertime
+
+- category: Topic
+  expansions:
+    - text: the quicksort algorithm
+    - text: stochastic gradient descent
+    - text: the Great Vowel Shift
+    - text: northern lights
+    - text: the Romantic period
+    - text: the Civil Rights movement
+    - text: the Pacific Northwest
+    - text: El Niño
+
+- category: TopicPair
+  expansions:
+    - text: north and south
+    - text: gradient descent and gradient ascent
+    - text: vowels and consonants
+    - text: C and C++
+    - text: Google and Microsoft
+    - text: US and Canada
+    - text: cats and dogs
+    - text: the Baroque period and the Romantic period
+
+- category: Style
+  expansions:
+    - text: as a paragraph.
+    - text: as a haiku.
+    - text: as a limerick.
+    - text: in the style of a Shakespeare sonnet.
+    - text: in the style of a court case.
+    - text: in the style of Snoop Dogg.
+    - text: so that a ${Age}-year old can understand it.
+    - text: in ${Language}.
+    - text: in 3 bullet points.
+    - text: in 8 bullet points.
+
+- category: Age
+  expansions:
+    - text: "5"
+    - text: "9"
+    - text: "13"
+
+- category: Language
+  expansions:
+    - text: Italian
+    - text: Greek
+    - text: Indian
+    - text: Chinese
+    - text: Thai

helm/benchmark/scenarios/conv_fin_qa_calc_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import Dict, List, Any
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     VALID_SPLIT,
     CORRECT_TAG,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_file_downloaded
 
@@ -95,3 +97,22 @@ class ConvFinQACalcScenario(Scenario):
                 for raw_instance in raw_instances:
                     instances.append(self.convert_to_instance(raw_instance, split))
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="conv_fin_qa_calc",
+            display_name="ConvFinQACalc",
+            short_display_name=None,
+            description="A mathematical calculation benchmark based on ConvFinQA: Exploring the Chain "
+            "of Numerical Reasoning in Conversational Finance Question Answering [(Chen ey "
+            "al., 2022)](https://arxiv.org/pdf/2210.03849.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="question answering with numeric reasoning",
+                what="financial reports",
+                when="1999 to 2019",
+                who="financial experts",
+                language="English",
+            ),
+            main_metric="float_equiv",
+            main_split="valid",
+        )

helm/benchmark/scenarios/copyright_scenario.py

@@ -3,8 +3,18 @@ import os
 import tqdm
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Reference, CORRECT_TAG, TEST_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 datatag2hash_text = {
     # The "average" book.
@@ -81,3 +91,27 @@ class CopyrightScenario(Scenario):
                 ),
             )
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.datatag in datatag2hash_code:
+            return ScenarioMetadata(
+                name="copyright_code",
+                display_name="Copyright (code)",
+                description="Scenario introduced in this work to measure copyright and memorization "
+                "behavior for code, based off of [Carlini et al. "
+                "(2021)](https://www.usenix.org/biblio-11958).",
+                taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+                main_metric="unknown",
+                main_split="test",
+            )
+        else:
+            return ScenarioMetadata(
+                name="copyright_text",
+                display_name="Copyright (text)",
+                description="Scenario introduced in this work to measure copyright and memorization "
+                "behavior for books, based off of [Carlini et al. "
+                "(2021)](https://www.usenix.org/biblio-11958).",
+                taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+                main_metric="unknown",
+                main_split="test",
+            )

helm/benchmark/scenarios/cti_to_mitre_scenario.py

@@ -6,6 +6,7 @@ from typing import Any, List, Dict
 import pandas as pd
 from pandas import DataFrame
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -16,6 +17,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -238,3 +240,22 @@ class CtiToMitreScenario(Scenario):
         # return all instances
         all_instances = instances_train + instances_test
         return all_instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cti_to_mitre",
+            display_name="CTI-to-MITRE Cyber Threat Intelligence",
+            short_display_name=None,
+            description="A classification benchmark based on Automatic Mapping of Unstructured Cyber "
+            "Threat Intelligence - An Experimental Study [(Orbinato et al., "
+            "2022)](https://arxiv.org/pdf/2208.12144.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="text classification",
+                what="Descriptions of malicious techniques",
+                when="Before 2022",
+                who="Security professionals",
+                language="English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/decodingtrust_adv_demonstration_scenario.py

@@ -1,9 +1,19 @@
 import os
 import json
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.common.general import ensure_directory_exists
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Reference, CORRECT_TAG, Output, VALID_SPLIT
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Reference,
+    CORRECT_TAG,
+    Output,
+    VALID_SPLIT,
+    ScenarioMetadata,
+)
 
 TASKS = {
     "counterfactual": ["snli_premise", "snli_hypothesis"]
@@ -167,3 +177,14 @@ class DecodingTrustAdvDemoScenario(Scenario):
             if self.demo_name in ["cf", "zero"]:
                 break
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="decodingtrust_adv_demonstration",
+            display_name="DecodingTrust - Adversarial Demonstrations",
+            short_display_name="AdvDemo",
+            description="Robustness analysis of LM generations when facing adversarial demonstrations",
+            taxonomy=TaxonomyInfo(task="text classification", what="?", when="?", who="?", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/decodingtrust_adv_robustness_scenario.py

@@ -1,8 +1,18 @@
 import os
 import json
 from typing import List, Dict
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, VALID_SPLIT, Reference, Output, CORRECT_TAG
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    VALID_SPLIT,
+    Reference,
+    Output,
+    CORRECT_TAG,
+    ScenarioMetadata,
+)
 
 TASK_DESCRIPTIONS = {
     "sst2": "For the given sentence, label the sentiment of the sentence as positive or negative. The answer should be "
@@ -119,3 +129,15 @@ class DecodingTrustAdvRobustnessScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="decodingtrust_adv_robustness",
+            display_name="DecodingTrust - AdvGLUE++",
+            short_display_name="AdvGLUE++",
+            description="Adversarial perturbations of the GLUE dataset generated against open-source "
+            "LLMs including Alpaca, Vicuna, and Stable-Vicuna",
+            taxonomy=TaxonomyInfo(task="text classification", what="?", when="?", who="?", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/decodingtrust_fairness_scenario.py

@@ -1,8 +1,18 @@
 import os
 import json
 from typing import List, Dict
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, VALID_SPLIT, Reference, Output, CORRECT_TAG
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    VALID_SPLIT,
+    Reference,
+    Output,
+    CORRECT_TAG,
+    ScenarioMetadata,
+)
 
 TASK_DESCRIPTIONS = {
     "predict_annual_income": "For each snippet of text, predict whether the person described has an annual income "
@@ -75,3 +85,14 @@ class DecodingTrustFairnessScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="decodingtrust_fairness",
+            display_name="DecodingTrust - Fairness",
+            short_display_name="Fairness",
+            description="Fairness analysis of LLMs",
+            taxonomy=TaxonomyInfo(task="text classification", what="?", when="?", who="?", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/decodingtrust_machine_ethics_scenario.py

@@ -4,8 +4,17 @@ import random
 import numpy as np
 import pandas as pd
 from typing import List, Dict, Tuple, Union
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Reference, Output, CORRECT_TAG
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Reference,
+    Output,
+    CORRECT_TAG,
+    ScenarioMetadata,
+)
 
 OPTION_DICT: Dict[str, Union[List[int], List[str]]] = {
     "ethics_commonsense_short": ["not wrong", "wrong"],
@@ -322,3 +331,14 @@ class DecodingTrustMachineEthicsScenario(Scenario):
         )
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="decodingtrust_machine_ethics",
+            display_name="DecodingTrust - Ethics",
+            short_display_name="Ethics",
+            description="Evaluation of the understanding of ethical behaviors of LLMs",
+            taxonomy=TaxonomyInfo(task="text classification", what="?", when="?", who="?", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/decodingtrust_ood_robustness_scenario.py

@@ -1,6 +1,7 @@
 import os
 import json
 from typing import List, Dict
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
     CORRECT_TAG,
+    ScenarioMetadata,
 )
 
 TASK = {
@@ -202,3 +204,14 @@ class DecodingTrustOODRobustnessScenario(Scenario):
             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="decodingtrust_adv_demonstration",
+            display_name="DecodingTrust - Adversarial Demonstrations",
+            short_display_name="AdvDemo",
+            description="Robustness analysis of LM generations when facing adversarial demonstrations",
+            taxonomy=TaxonomyInfo(task="text classification", what="?", when="?", who="?", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/decodingtrust_privacy_scenario.py

@@ -4,8 +4,9 @@ import string
 import random
 from tqdm import tqdm
 from typing import List, Dict, Optional
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Reference, Output, TEST_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Reference, Output, TEST_SPLIT, ScenarioMetadata
 
 option_keys = ["pii", "privacy_understanding", "enron_email_extraction"]
 
@@ -217,6 +218,17 @@ class DecodingTrustPrivacyScenario(Scenario):
 
         return instances
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="decodingtrust_privacy",
+            display_name="DecodingTrust - Privacy",
+            short_display_name="Privacy",
+            description="Evaluation of the privacy understanding and privacy preserving properties of " "LLMs",
+            taxonomy=TaxonomyInfo(task="text classification", what="?", when="?", who="?", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )
+
 
 def get_local_domain(email):
     return email.split("@")

helm/benchmark/scenarios/decodingtrust_stereotype_bias_scenario.py

@@ -1,7 +1,8 @@
 import json
 import os
 from typing import List, Dict
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, ScenarioMetadata
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import Reference, Output
 
@@ -66,3 +67,14 @@ class DecodingTrustStereotypeBiasScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="decodingtrust_stereotype_bias",
+            display_name="DecodingTrust - Stereotype Bias",
+            short_display_name="Stereotype",
+            description="Manually crafted stereotype user prompts from DecodingTrust",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="unknown",
+            main_split="test",
+        )