crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/codeinsights_student_coding_scenario.py

@@ -0,0 +1,162 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsStudentCodingScenario(Scenario):
+    name = "codeinsights_student_coding"
+    description = "Mimic student C++ style on foundational questions"
+    tags = ["codeinsights", "c++", "student_coding"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario1_2_data.csv")
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        instances = []
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values("timestamp")
+            if len(student_df) < 4:
+                continue
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Get test cases for this question
+            question_id = target.get("question_unittest_id", None)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass (0 or 1) for the target question
+            student_correctness_pattern = target.get("pass", None)
+            main_part = int(student_correctness_pattern)  # "1111111111"
+            # Convert each character to an int
+            student_correctness_list = [int(ch) for ch in str(main_part)]  # [1,1,1,1,1,1,1,1,1,1]
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            prompt = (
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                f"Week: {target['week']}\n"
+                f"Topic: {target['topic']}\n\n"
+                "Example 1:\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Your Code:\n"
+                f"{first['response']}\n\n"
+                "Example 2:\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Your Code:\n"
+                f"{second['response']}\n\n"
+                "Example 3:\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Your Code:\n"
+                f"{third['response']}\n\n"
+                "Now, using that same student style, attempt this:\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code includes any class definition when needed. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/codeinsights_student_mistake_scenario.py

@@ -0,0 +1,188 @@
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, Output, Reference, VALID_SPLIT, CORRECT_TAG
+import pandas as pd
+import requests
+
+
+class CodeInsightsStudentMistakeScenario(Scenario):
+    name = "codeinsights_student_mistake"
+    description = "Mimic how students mistake their C++ codes on foundational questions"
+    tags = ["codeinsights", "c++", "student_mistake"]
+
+    def __init__(self, num_testcases: int = 1):
+        super().__init__()
+        self.num_testcases = num_testcases
+
+    def get_instances(self, output_path: str):
+        df = pd.read_csv("https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/Scenario3_data.csv")
+        student_topic = pd.read_csv(
+            "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/student_performace_by_topic.csv"
+        )
+
+        instances = []
+        for student_id, student_df in df.groupby("student_id"):
+            student_df = student_df.sort_values(by=["student_id", "question_unittest_id", "timestamp"])
+            if len(student_df) < 4:
+                continue
+            first = student_df.iloc[0]
+            second = student_df.iloc[1]
+            third = student_df.iloc[2]
+            target = student_df.iloc[3]
+
+            # Get test cases for this question
+            question_id = target.get("question_unittest_id", None)
+            question_test_cases = []
+            tc_parsing_success = True
+
+            for testcase_str in target["question_unittests"].split("Unittest")[1:]:
+                testcase_str = testcase_str[testcase_str.find(":") + 1 :]
+                input_idx = testcase_str.find("Input:")
+                std_in_idx = testcase_str.find("STD input:")
+                output_idx = testcase_str.find("Output:")
+                if input_idx == -1 or std_in_idx == -1 or output_idx == -1:
+                    tc_parsing_success = False
+                    break
+
+                testcase = {
+                    "input": testcase_str[input_idx + 6 : std_in_idx].strip(),
+                    "std_in": testcase_str[std_in_idx + 10 : output_idx].strip(),
+                    "output": testcase_str[output_idx + 7 :].strip(),
+                }
+                question_test_cases.append(testcase)
+
+            if not tc_parsing_success:
+                continue
+
+            if len(question_test_cases) < self.num_testcases:
+                # If not enough test cases, skip this question
+                continue
+            if self.num_testcases >= 0:
+                # If more than one test case is requested, only take the first ones
+                question_test_cases = question_test_cases[: self.num_testcases]
+
+            # Get student pass (0 or 1) for the target question
+            student_correctness_pattern = target.get("pass", None)
+            main_part = int(student_correctness_pattern)  # "1111111111"
+            # Convert each character to an int
+            student_correctness_list = [int(ch) for ch in str(main_part)]  # [1,1,1,1,1,1,1,1,1,1]
+
+            # Student specific topic performance in previous attempts
+            student_level_prompt = f"Student {student_id} has the following performance across topics:\n"
+            topic_performance = student_topic[student_topic["student_id"] == student_id]
+            for _, row in topic_performance.iterrows():
+                topic = row["topic"]
+                pass_rate = round(row["pass_rate"], 2)
+                perfect = round(row["perfect"], 2)
+
+                student_level_prompt += (
+                    f"- For topic '{topic}', the unit test pass rate is {pass_rate}, "
+                    f"and the rate of passing all unit tests is {perfect}.\n"
+                )
+
+            prompt = (
+                "=== Student Profile ===\n"
+                f"{student_level_prompt}\n"
+                "When students submit a code to the platform, it will be tested by number of unit tests, where"
+                "- Unit test pass rate = proportion of unit tests passed with the code \n"
+                "- Full pass rate   = proportion of code passing all unit tests\n\n"
+                "=== Past Mistake Examples ===\n"
+                "Example 1 (Week {first['week']}, Topic: {first['topic']}):\n"
+                f"Question: {first['question_name']} — {first['question_text']}\n"
+                "Template:\n"
+                f"{first['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{first['response_mistake']}\n\n"
+                "Example 2 (Week {second['week']}, Topic: {second['topic']}):\n"
+                f"Question: {second['question_name']} — {second['question_text']}\n"
+                "Template:\n"
+                f"{second['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{second['response_mistake']}\n\n"
+                "Example 3 (Week {third['week']}, Topic: {third['topic']}):\n"
+                f"Question: {third['question_name']} — {third['question_text']}\n"
+                "Template:\n"
+                f"{third['question_template']}\n"
+                "Student's Response Code with Error:\n"
+                f"{third['response_mistake']}\n\n"
+                "=== New Target Problem ===\n"
+                f"Week: {target['week']}, Topic: {target['topic']}\n"
+                f"Question: {target['question_name']} — {target['question_text']}\n"
+                f"Unit Test Input: {question_test_cases}\n\n"
+                if question_test_cases
+                else ""
+                "Template:\n"
+                f"{target['question_template']}\n\n"
+                "⚠**Instructions:**\n"
+                "1. Mimic your own coding style, naming conventions, indentation, and typical error patterns.\n"
+                "2. Introduce mistake you are likely to make (e.g., off‐by‐one index, wrong initialization, "
+                "missing edge case).\n"
+                "3. Do **not** produce a fully correct solution or add unfamiliar optimizations.\n\n"
+                "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template. "
+                "DO NOT reproduce the template part as the generated code would be inserted to the template, "
+                "and make sure the code is compatible with the Unit Test Input. "
+                "int main() is always declared already so DO NOT produce that initialization on the code. "
+                "Ensure your code is includes any class definition when needed. "
+                "Return the code in C++ code block format, and nothing else."
+            )
+
+            print(f"\n=== DEBUG INFO FOR STUDENT {student_id}, QUESTION {question_id} ===")
+            print(f"Test cases loaded: {len(question_test_cases)}")
+            print(f"Student correctness pattern: {student_correctness_list}")
+            print(f"Original pass field: {target.get('pass', 'MISSING')}")
+            print(f"Question template exists: {'question_template' in target}")
+            print(f"Question name: {target.get('question_name', 'MISSING')}")
+
+            # Also add this validation in your UnitTestAlignmentMetric evaluate_generation method:
+            def evaluate_generation(self, adapter_spec, request_state, metric_service, eval_cache_path):
+                print("\n=== UNIT TEST METRIC DEBUG ===")
+                print(f"Has extra_data: {hasattr(request_state.instance, 'extra_data')}")
+                if hasattr(request_state.instance, "extra_data"):
+                    extra_data = request_state.instance.extra_data
+                    print(f"Extra data keys: {list(extra_data.keys())}")
+                    print(f"Test cases: {len(extra_data.get('test_cases', []))}")
+                    print(f"Student pattern: {extra_data.get('student_correctness_pattern', 'MISSING')}")
+
+            instances.append(
+                Instance(
+                    id=f"{student_id}_{target['question_unittest_id']}",
+                    input=Input(text=prompt),
+                    references=[Reference(output=Output(text=target["response_mistake"]), tags=[CORRECT_TAG])],
+                    extra_data={
+                        "question_template": target["question_template"],
+                        "test_cases": question_test_cases,
+                        "question_id": str(question_id) if question_id else None,
+                        "question_name": target.get("question_name", ""),
+                        "student_id": str(student_id),
+                        "student_correctness_pattern": student_correctness_list,
+                    },
+                    split=VALID_SPLIT,
+                )
+            )
+        return instances
+
+    def _load_test_cases(self):
+        """
+        Load test cases from external source or return None if not available.
+        This method should be implemented based on where your test cases are stored.
+
+        Expected format:
+        {
+            "question_id": [
+                {
+                    "unittest": "test_id",
+                    "input": "test input code",
+                    "output": "expected output"
+                },
+                ...
+            ],
+            ...
+        }
+        """
+        try:
+            response = requests.get(
+                "https://huggingface.co/datasets/Kazchoko/my_dataset/resolve/main/test_cases_by_qid.json"
+            )
+            if response.status_code == 200:
+                return response.json()
+        except Exception as e:
+            print(f"Failed to load test cases from URL: {e}")
+            return {}

helm/benchmark/scenarios/commonsense_scenario.py

@@ -2,9 +2,10 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.hierarchical_logger import hlog
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,
@@ -14,6 +15,7 @@ from .scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -72,6 +74,19 @@ class HellaSwagScenario(Scenario):
         assert len(answers) == 4
         return _make_instance(question=question, answers=answers, correct_answer=correct_answer, split=split)
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="hellaswag",
+            display_name="HellaSwag",
+            description="The HellaSwag benchmark for commonsense reasoning in question answering "
+            "[(Zellers et al., 2019)](https://aclanthology.org/P19-1472/).",
+            taxonomy=TaxonomyInfo(
+                task="question answering", what="commonsense reasoning", when="?", who="?", language="English"
+            ),
+            main_metric="exact_match",
+            main_split="valid",
+        )
+
 
 class OpenBookQA(Scenario):
     name = "openbookqa"
@@ -113,6 +128,23 @@ class OpenBookQA(Scenario):
         assert item["question"]["choices"][correct_choice]["label"] == item["answerKey"]
         return _make_instance(question=question, answers=answers, correct_answer=correct_answer, split=split)
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="openbookqa",
+            display_name="OpenbookQA",
+            description="The OpenbookQA benchmark for commonsense-intensive open book question "
+            "answering [(Mihaylov et al., 2018)](https://aclanthology.org/D18-1260/).",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="elementary science",
+                when="2018",
+                who="Amazon Mechnical Turk workers",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CommonSenseQAScenario(Scenario):
     name = "commonsenseqa"

helm/benchmark/scenarios/compositional_instructions.yaml

@@ -0,0 +1,70 @@
+rules:
+- category: Root
+  expansions:
+    - text: ${Task} ${Style}
+
+- category: Task
+  expansions:
+    - text: Explain ${HowTo}
+    - text: Explain ${Topic}
+    - text: Tell me a joke about ${Topic}
+    - text: Tell me a joke about ${TopicPair}
+    - text: What are the similarities between ${TopicPair}
+    - text: What are the differences between ${TopicPair}
+    - text: Tell me 5 surprising facts about ${Topic}
+    - text: Persuade me to learn about ${Topic}
+
+- category: HowTo
+  expansions:
+    - text: how airplanes fly
+    - text: how maglev trains work
+    - text: how to grow tomatoes in the wintertime
+
+- category: Topic
+  expansions:
+    - text: the quicksort algorithm
+    - text: stochastic gradient descent
+    - text: the Great Vowel Shift
+    - text: northern lights
+    - text: the Romantic period
+    - text: the Civil Rights movement
+    - text: the Pacific Northwest
+    - text: El Niño
+
+- category: TopicPair
+  expansions:
+    - text: north and south
+    - text: gradient descent and gradient ascent
+    - text: vowels and consonants
+    - text: C and C++
+    - text: Google and Microsoft
+    - text: US and Canada
+    - text: cats and dogs
+    - text: the Baroque period and the Romantic period
+
+- category: Style
+  expansions:
+    - text: as a paragraph.
+    - text: as a haiku.
+    - text: as a limerick.
+    - text: in the style of a Shakespeare sonnet.
+    - text: in the style of a court case.
+    - text: in the style of Snoop Dogg.
+    - text: so that a ${Age}-year old can understand it.
+    - text: in ${Language}.
+    - text: in 3 bullet points.
+    - text: in 8 bullet points.
+
+- category: Age
+  expansions:
+    - text: "5"
+    - text: "9"
+    - text: "13"
+
+- category: Language
+  expansions:
+    - text: Italian
+    - text: Greek
+    - text: Indian
+    - text: Chinese
+    - text: Thai

helm/benchmark/scenarios/conv_fin_qa_calc_scenario.py

@@ -0,0 +1,118 @@
+import json
+import os
+from typing import Dict, List, Any
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Output,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_file_downloaded
+
+
+class ConvFinQACalcScenario(Scenario):
+    """A mathematical calculation benchmark based on ConvFinQA.
+
+    Data source:
+    https://github.com/czyssrs/ConvFinQA
+
+    Reference:
+    Zhiyu Chen, Shiyang Li, Charese Smiley, Zhiqiang Ma, Sameena Shah, and William Yang Wang. 2022.
+    ConvFinQA: Exploring the Chain of Numerical Reasoning in Conversational Finance Question Answering.
+    In Proceedings of the 2022 Conference on Empirical Methods in Natural Language Processing,
+    pages 6279–6292, Abu Dhabi, United Arab Emirates. Association for Computational Linguistics.
+    https://aclanthology.org/2022.emnlp-main.421
+    """  # noqa: E501
+
+    name = "conv_fin_qa_calc"
+    description = "A mathematical calculation benchmark based on ConvFinQA: Exploring the Chain of Numerical Reasoning in Conversational Finance Question Answering [(Chen ey al., 2022)](https://arxiv.org/pdf/2210.03849.pdf)."  # noqa: E501
+    tags = ["question_answering", "finance"]
+
+    DATASET_DOWNLOAD_URL: str = (
+        "https://github.com/czyssrs/ConvFinQA/raw/cf3eed2d5984960bf06bb8145bcea5e80b0222a6/data.zip"
+    )
+
+    _SPLIT_TO_JSON_FILE_NAME: Dict[str, str] = {TRAIN_SPLIT: "train_turn.json", VALID_SPLIT: "dev_turn.json"}
+
+    def make_pseudo_markdown_table(self, table: List[List[Any]], sep: str = "\n") -> str:
+        markdown_lines: List[str] = []
+
+        for row in table:
+            row_inner_markdown = " | ".join([str(cell) for cell in row])
+            row_markdown = f"| {row_inner_markdown} |"
+            markdown_lines.append(row_markdown)
+
+        return sep.join(markdown_lines)
+
+    def convert_to_instance(self, dic: Dict[str, Any], split: str, sep: str = "\n") -> Instance:
+        linearized_table = self.make_pseudo_markdown_table(dic["table"])
+        input_text = f"Table: {sep}{linearized_table}{sep}{sep}"
+
+        if "gold_ind" in dic["annotation"]:
+            facts = dic["annotation"]["gold_ind"]
+        elif "gold_inds" in dic["annotation"]:
+            facts = dic["annotation"]["gold_inds"]
+        else:
+            facts = {}
+        table_text = ""
+        for fact_type, fact in facts.items():
+            if "text" in fact_type:
+                table_text += fact
+        if table_text:
+            input_text += f"Text: {sep}{table_text}{sep}{sep}"
+
+        for ind, q in enumerate(dic["annotation"]["cur_dial"]):
+            if ind < len(dic["annotation"]["cur_dial"]) - 1:
+                input_text += f"Question: {q}{sep}Answer: {dic['annotation']['exe_ans_list'][ind]}{sep}"
+            else:
+                input_text += f"Question: {q}"
+
+        answer = str(dic["annotation"]["exe_ans"])
+        return Instance(
+            input=Input(text=input_text),
+            references=[Reference(Output(text=answer), tags=[CORRECT_TAG])],
+            split=split,
+        )
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = os.path.join(output_path, "data")
+        ensure_file_downloaded(
+            source_url=self.DATASET_DOWNLOAD_URL,
+            target_path=os.path.join(output_path, "data"),
+            unpack=True,
+            unpack_type="unzip",
+        )
+        instances: List[Instance] = []
+        for split, json_file_name in self._SPLIT_TO_JSON_FILE_NAME.items():
+            json_file_path = os.path.join(data_path, json_file_name)
+            with open(json_file_path) as f:
+                raw_instances = json.load(f)
+                for raw_instance in raw_instances:
+                    instances.append(self.convert_to_instance(raw_instance, split))
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="conv_fin_qa_calc",
+            display_name="ConvFinQACalc",
+            short_display_name=None,
+            description="A mathematical calculation benchmark based on ConvFinQA: Exploring the Chain "
+            "of Numerical Reasoning in Conversational Finance Question Answering [(Chen ey "
+            "al., 2022)](https://arxiv.org/pdf/2210.03849.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="question answering with numeric reasoning",
+                what="financial reports",
+                when="1999 to 2019",
+                who="financial experts",
+                language="English",
+            ),
+            main_metric="float_equiv",
+            main_split="valid",
+        )

helm/benchmark/scenarios/copyright_scenario.py

@@ -3,8 +3,18 @@ import os
 import tqdm
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TEST_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 datatag2hash_text = {
     # The "average" book.
@@ -81,3 +91,27 @@ class CopyrightScenario(Scenario):
                 ),
             )
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.datatag in datatag2hash_code:
+            return ScenarioMetadata(
+                name="copyright_code",
+                display_name="Copyright (code)",
+                description="Scenario introduced in this work to measure copyright and memorization "
+                "behavior for code, based off of [Carlini et al. "
+                "(2021)](https://www.usenix.org/biblio-11958).",
+                taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+                main_metric="unknown",
+                main_split="test",
+            )
+        else:
+            return ScenarioMetadata(
+                name="copyright_text",
+                display_name="Copyright (text)",
+                description="Scenario introduced in this work to measure copyright and memorization "
+                "behavior for books, based off of [Carlini et al. "
+                "(2021)](https://www.usenix.org/biblio-11958).",
+                taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+                main_metric="unknown",
+                main_split="test",
+            )

helm/benchmark/scenarios/covid_dialog_scenario.py

@@ -2,7 +2,16 @@ import os
 from typing import List
 
 from helm.common.general import ensure_directory_exists, ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, ALL_SPLITS, CORRECT_TAG, VALID_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    ALL_SPLITS,
+    CORRECT_TAG,
+    VALID_SPLIT,
+    Input,
+    Output,
+)
 
 
 class COVIDDialogScenario(Scenario):