crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/cleva_scenario.py

@@ -10,6 +10,7 @@ from helm.benchmark.adaptation.adapters.adapter_factory import (
     ADAPT_MULTIPLE_CHOICE_SEPARATE_ORIGINAL,
     ADAPT_GENERATION,
 )
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.runner import get_benchmark_output_path
 from helm.common.general import (
     assert_is_str,
@@ -18,7 +19,7 @@ from helm.common.general import (
     ensure_directory_exists,
 )
 from helm.common.hierarchical_logger import hlog
-from .scenario import (
+from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
     Reference,
@@ -27,9 +28,10 @@ from .scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
     get_scenario_cache_path,
 )
-from .code_scenario import CodeReference, CodeInstance
+from helm.benchmark.scenarios.code_scenario import CodeReference, CodeInstance
 
 
 CLEVA_DATA_URL = "http://39.108.215.175/data"
@@ -576,6 +578,19 @@ class CLEVATextClassificationScenario(CLEVAScenario):
     def task(self) -> str:
         return "text_classification"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_text_classification",
+            display_name="CLEVA (Chinese) text classification",
+            description="This scenario has two subtasks. Classify if an utterance is humorous and "
+            "identify news topic based on its title.",
+            taxonomy=TaxonomyInfo(
+                task="text classification", what="news or chitchat", when="2010s", who="n/a", language="Chinese"
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAOpinionMiningScenario(CLEVAScenario):
     """
@@ -597,6 +612,18 @@ class CLEVAOpinionMiningScenario(CLEVAScenario):
     def task(self) -> str:
         return "opinion_mining"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_opinion_mining",
+            display_name="CLEVA (Chinese) opinion mining",
+            description="Extract the target of an opinion.",
+            taxonomy=TaxonomyInfo(
+                task="opinion target extraction", what="n/a", when="2010s", who="n/a", language="Chinese"
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAPinyinTransliterationScenario(CLEVAScenario):
     """
@@ -626,6 +653,22 @@ class CLEVAPinyinTransliterationScenario(CLEVAScenario):
     def task(self) -> str:
         return "pinyin_transliteration"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_pinyin_transliteration",
+            display_name="CLEVA (Chinese) pinyin transliteration",
+            description="Scenario that asks the model to translate between Chinese and Pinyin.",
+            taxonomy=TaxonomyInfo(
+                task="pinyin transliteration",
+                what="n/a",
+                when="2023",
+                who="automatically generated by algorithm",
+                language="Chinese, Pinyin",
+            ),
+            main_metric="chinese_bleu_1",
+            main_split="test",
+        )
+
 
 class CLEVAClassicalChineseUnderstandingScenario(CLEVAScenario):
     """
@@ -651,6 +694,20 @@ class CLEVAClassicalChineseUnderstandingScenario(CLEVAScenario):
     def task(self) -> str:
         return "classical_chinese_understanding"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_classical_chinese_understanding",
+            display_name="CLEVA (Chinese) classical Chinese understanding",
+            description="Scenario for evaluating the understanding of classical Chinese by selecting "
+            "the appropriate classical Chinese translation for a given modern Chinese "
+            "sentence.",
+            taxonomy=TaxonomyInfo(
+                task="translation", what="n/a", when="2021 or before", who="n/a", language="Classical Chinese"
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVASentimentAnalysisScenario(CLEVAScenario):
     """
@@ -674,6 +731,22 @@ class CLEVASentimentAnalysisScenario(CLEVAScenario):
     def task(self) -> str:
         return "sentiment_analysis"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_sentiment_analysis",
+            display_name="Sentiment analysis",
+            description="Chinese sentiment analysis for product reviews.",
+            taxonomy=TaxonomyInfo(
+                task="sentiment analysis",
+                what="product reviews",
+                when="2021 or before",
+                who="customers",
+                language="Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAInstructionFollowingScenario(CLEVAScenario):
     """
@@ -702,6 +775,24 @@ class CLEVAInstructionFollowingScenario(CLEVAScenario):
     def task(self) -> str:
         return "instruction_following"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_instruction_following",
+            display_name="CLEVA (Chinese) instruction following",
+            description="Scenario that examines whether models could follow human instructions, mainly "
+            "uncommon ones. It contains two subtasks: 'redefine' and "
+            "'pattern_matching_suppression'.",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="natural language questions",
+                when="2023",
+                who="automatically generated from templates",
+                language="synthetic",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAFactCheckingScenario(CLEVAScenario):
     """
@@ -726,6 +817,23 @@ class CLEVAFactCheckingScenario(CLEVAScenario):
     def task(self) -> str:
         return "fact_checking"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_fact_checking",
+            display_name="CLEVA (Chinese) fact checking",
+            description="Scenario that lets models identify whether the given fact is true to test "
+            "their factuality.",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="factual statements in natural language form",
+                when="2022 or before",
+                who="n/a",
+                language="Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVATranslationScenario(CLEVAScenario):
     """
@@ -758,6 +866,18 @@ class CLEVATranslationScenario(CLEVAScenario):
     def task(self) -> str:
         return "translation"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_translation",
+            display_name="Translation",
+            description="Scenario for measuring the translation quality between Chinese and English.",
+            taxonomy=TaxonomyInfo(
+                task="translation", what="news", when="2022 or before", who="n/a", language="Chinese, English"
+            ),
+            main_metric="cleva_machine_translation_bleu",
+            main_split="test",
+        )
+
 
 class CLEVAToxicityDetectionScenario(CLEVAScenario):
     """
@@ -781,6 +901,22 @@ class CLEVAToxicityDetectionScenario(CLEVAScenario):
     def task(self) -> str:
         return "toxicity_detection"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_toxicity_detection",
+            display_name="CLEVA (Chinese) toxicity detection",
+            description="Ask models about the offensiveness of the given text.",
+            taxonomy=TaxonomyInfo(
+                task="toxicity classification",
+                what="text from Chinese social media",
+                when="2022 or before",
+                who="web users",
+                language="Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAParaphraseGenerationScenario(CLEVAScenario):
     """
@@ -814,6 +950,18 @@ class CLEVAParaphraseGenerationScenario(CLEVAScenario):
         )
         return instance
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_paraphrase_generation",
+            display_name="CLEVA (Chinese) paraphrase generation",
+            description="Generate a paraphrase of a given sentence.",
+            taxonomy=TaxonomyInfo(
+                task="paraphrase generation", what="n/a", when="2010s", who="n/a", language="Chinese"
+            ),
+            main_metric="chinese_ibleu",
+            main_split="test",
+        )
+
 
 class CLEVAIntentUnderstandingScenario(CLEVAScenario):
     """
@@ -848,6 +996,19 @@ class CLEVAIntentUnderstandingScenario(CLEVAScenario):
     def task(self) -> str:
         return "intent_understanding"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_intent_understanding",
+            display_name="CLEVA (Chinese) intent understanding",
+            description="Tests whether the model could capture the writing intention of the authors "
+            "after reading an article.",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering", what="exam", when="1990-2022", who="n/a", language="Chinese"
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVACoreferenceResolutionScenario(CLEVAScenario):
     """
@@ -870,6 +1031,23 @@ class CLEVACoreferenceResolutionScenario(CLEVAScenario):
     def task(self) -> str:
         return "coreference_resolution"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_coreference_resolution",
+            display_name="CLEVA (Chinese) coreference resolution",
+            description="Scenario for testing models on solving coreference resolution problems (the "
+            "winograd schema challenge).",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="contemporary Chinese literary works",
+                when="2020 or before",
+                who="n/a",
+                language="Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAReadingComprehensionScenario(CLEVAScenario):
     """
@@ -898,6 +1076,18 @@ class CLEVAReadingComprehensionScenario(CLEVAScenario):
     def task(self) -> str:
         return "reading_comprehension"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_reading_comprehension",
+            display_name="Reading comprehension",
+            description="Answer a multiple-choice question based on a given paragraph.",
+            taxonomy=TaxonomyInfo(
+                task="question answering", what="n/a", when="2019 or before", who="n/a", language="Chinese"
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVADialogueGenerationScenario(CLEVAScenario):
     """
@@ -964,6 +1154,22 @@ class CLEVADialogueGenerationScenario(CLEVAScenario):
 
         return instances
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_dialogue_generation",
+            display_name="CLEVA (Chinese) dialogue generation",
+            description="Task-oriented dialogue between a user and a system.",
+            taxonomy=TaxonomyInfo(
+                task="dialogue generation",
+                what="task-oriented dialogue on hotel, restaurant, attraction, metro, " "and taxi domain",
+                when="2020 or before",
+                who="user and assistant",
+                language="Chinese",
+            ),
+            main_metric="chinese_bleu_1",
+            main_split="test",
+        )
+
 
 class CLEVASubjectKnowledgeScenario(CLEVAScenario):
     """
@@ -993,6 +1199,24 @@ class CLEVASubjectKnowledgeScenario(CLEVAScenario):
     def task(self) -> str:
         return "subject_knowledge"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_subject_knowledge",
+            display_name="CLEVA (Chinese) subject knowledge",
+            description="Scenario inspired by [Petroni et al. "
+            "(2019)](https://aclanthology.org/D19-1250/) to extensively test factual "
+            "knowledge in Chinese. It contains 13 subjects and a general domain.",
+            taxonomy=TaxonomyInfo(
+                task="knowledge base completion",
+                what="entity-relation-entity triples in natural language form",
+                when="2022 or before",
+                who="automatically generated from templates",
+                language="structured Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVACulturalKnowledgeScenario(CLEVAScenario):
     """
@@ -1022,6 +1246,23 @@ class CLEVACulturalKnowledgeScenario(CLEVAScenario):
     def task(self) -> str:
         return "cultural_knowledge"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_cultural_knowledge",
+            display_name="CLEVA (Chinese) cultural knowledge",
+            description="Scenario for evaluating models' understanding of Chinese culture. It has a "
+            "Chinese-idiom-focused subtask.",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="Various passages containing Chinese idioms",
+                when="2010s",
+                who="n/a",
+                language="Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAParaphraseIdentificationScenario(CLEVAScenario):
     """
@@ -1057,6 +1298,23 @@ class CLEVAParaphraseIdentificationScenario(CLEVAScenario):
     def task(self) -> str:
         return "paraphrase_identification"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_paraphrase_identification",
+            display_name="Paraphrase identification",
+            description="Identify if two sentences, from a dialogue or from the finance domain, share "
+            "the same meaning.",
+            taxonomy=TaxonomyInfo(
+                task="paraphrase identification",
+                what="financial questions or chitchat",
+                when="2020 or before",
+                who="n/a",
+                language="Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAClosedBookQuestionAnsweringScenario(CLEVAScenario):
     """
@@ -1117,6 +1375,24 @@ class CLEVAClosedBookQuestionAnsweringScenario(CLEVAScenario):
     def task(self) -> str:
         return "closed_book_question_answering"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_closed_book_question_answering",
+            display_name="CLEVA (Chinese) closed book question answering",
+            description="Closed-book question answering task comprises three subtasks. One is for the "
+            "medical domain, another for open-domain, and the last measures if a model "
+            "generates truthful answers.",
+            taxonomy=TaxonomyInfo(
+                task="question answering",
+                what="medical, open-domain, or truthful qa",
+                when="2022 or before",
+                who="n/a",
+                language="Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVASummarizationScenario(CLEVAScenario):
     """
@@ -1151,6 +1427,22 @@ class CLEVASummarizationScenario(CLEVAScenario):
     def task(self) -> str:
         return "summarization"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_summarization",
+            display_name="CLEVA (Chinese) summarization",
+            description="Summarize a dialogue between a customer representative and a customer.",
+            taxonomy=TaxonomyInfo(
+                task="summarization",
+                what="e-commerce dialogues",
+                when="2021 or before",
+                who="customers and representatives",
+                language="Chinese",
+            ),
+            main_metric="chinese_rouge_2",
+            main_split="test",
+        )
+
 
 class CLEVABiasScenario(CLEVAScenario):
     """
@@ -1215,6 +1507,23 @@ class CLEVABiasScenario(CLEVAScenario):
     def task(self) -> str:
         return "bias"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_bias",
+            display_name="CLEVA (Chinese) bias",
+            description="Scenario that gauges bias of four demographic categories in dialogues, "
+            "including race, gender, region, and occupation.",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="short Chinese dialogues from social media",
+                when="2022 or before",
+                who="web users",
+                language="Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVACopyrightScenario(CLEVAScenario):
     """
@@ -1231,6 +1540,20 @@ class CLEVACopyrightScenario(CLEVAScenario):
     def task(self) -> str:
         return "copyright"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_copyright",
+            display_name="CLEVA (Chinese) copyright",
+            description="Scenario that measures copyright and memorization behavior for Chinese books "
+            "and code, based off of [Carlini et al. "
+            "(2021)](https://www.usenix.org/biblio-11958).",
+            taxonomy=TaxonomyInfo(
+                task="next-word prediction", what="books and code", when="2023 or before", who="n/a", language="Chinese"
+            ),
+            main_metric="unknown",
+            main_split="test",
+        )
+
 
 class CLEVAConceptualGeneralizationScenario(CLEVAScenario):
     """
@@ -1257,6 +1580,23 @@ class CLEVAConceptualGeneralizationScenario(CLEVAScenario):
     def task(self) -> str:
         return "conceptual_generalization"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_conceptual_generalization",
+            display_name="CLEVA (Chinese) conceptual generalization",
+            description="Scenario that assesses whether models could generalize physical relations to a "
+            "synthetic grid world.",
+            taxonomy=TaxonomyInfo(
+                task="next-word prediction",
+                what="n/a",
+                when="2023",
+                who="automatically generated by algorithm",
+                language="synthetic",
+            ),
+            main_metric="cleva_top1_accuracy",
+            main_split="test",
+        )
+
 
 class CLEVACommonsenseReasoningScenario(CLEVAScenario):
     """
@@ -1289,6 +1629,23 @@ class CLEVACommonsenseReasoningScenario(CLEVAScenario):
     def task(self) -> str:
         return "commonsense_reasoning"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_commonsense_reasoning",
+            display_name="CLEVA (Chinese) commonsense reasoning",
+            description="Scenario that tests models' commonsense reasoning ability. There are two "
+            "subtasks: textual entailment and commonsense question answering.",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="n/a",
+                when="2023 or before",
+                who="n/a",
+                language="Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVADeductiveReasoningScenario(CLEVAScenario):
     """
@@ -1314,6 +1671,23 @@ class CLEVADeductiveReasoningScenario(CLEVAScenario):
     def task(self) -> str:
         return "deductive_reasoning"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_deductive_reasoning",
+            display_name="CLEVA (Chinese) deductive reasoning",
+            description="Scenario that gauges model's ability to reason deductive arguments. It "
+            "includes a modus tollens subtask.",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="natural language questions",
+                when="2023",
+                who="n/a",
+                language="structured Chinese",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAMathematicalCalculationScenario(CLEVAScenario):
     """
@@ -1348,6 +1722,24 @@ class CLEVAMathematicalCalculationScenario(CLEVAScenario):
     def task(self) -> str:
         return "mathematical_calculation"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_mathematical_calculation",
+            display_name="CLEVA (Chinese) mathematical calculation",
+            description="Scenario that evaluates the calculation ability of models. It has four "
+            "subtasks: three-digit addition, three-digit subtraction, two-digit "
+            "multiplication, and significant figures.",
+            taxonomy=TaxonomyInfo(
+                task="next-word prediction",
+                what="natural language math questions or pure math expressions",
+                when="2023",
+                who="automatically generated from templates",
+                language="synthetic",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAInductiveReasoningScenario(CLEVAScenario):
     """
@@ -1372,6 +1764,23 @@ class CLEVAInductiveReasoningScenario(CLEVAScenario):
     def task(self) -> str:
         return "inductive_reasoning"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_inductive_reasoning",
+            display_name="CLEVA (Chinese) inductive reasoning",
+            description="Scenario that tests models' ability to conclude rules from demonstrations and "
+            "apply them to unseen test instances.",
+            taxonomy=TaxonomyInfo(
+                task="next-word prediction",
+                what="n/a",
+                when="2023",
+                who="automatically generated by algorithm",
+                language="synthetic",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAReasoningPrimitiveScenario(CLEVAScenario):
     """
@@ -1427,6 +1836,23 @@ class CLEVAReasoningPrimitiveScenario(CLEVAScenario):
     def task(self) -> str:
         return "reasoning_primitive"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_reasoning_primitive",
+            display_name="CLEVA (Chinese) reasoning primitive",
+            description="Scenario focused on primitive reasoning, including dyck language continuation, "
+            "variable substitution, pattern induction, and pattern matching.",
+            taxonomy=TaxonomyInfo(
+                task="next-word prediction",
+                what="n/a",
+                when="2023",
+                who="automatically generated from templates",
+                language="synthetic",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVADataToTextGenerationScenario(CLEVAScenario):
     """
@@ -1470,6 +1896,19 @@ class CLEVADataToTextGenerationScenario(CLEVAScenario):
     def task(self) -> str:
         return "data_to_text_generation"
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_data_to_text_generation",
+            display_name="CLEVA (Chinese) data to text generation",
+            description="Generate a product description based on structured data containing various "
+            "product properties.",
+            taxonomy=TaxonomyInfo(
+                task="data-to-text generation", what="product description", when="2010s", who="n/a", language="Chinese"
+            ),
+            main_metric="chinese_bleu_1",
+            main_split="test",
+        )
+
 
 class CLEVAMathematicalReasoningScenario(CLEVAScenario):
     """
@@ -1521,6 +1960,20 @@ class CLEVAMathematicalReasoningScenario(CLEVAScenario):
             )
         return instance
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_mathematical_reasoning",
+            display_name="CLEVA (Chinese) mathematical reasoning",
+            description="Scenario that tests models' mathematical reasoning ability with "
+            "chain-of-thought style reasoning. It contains a math word problem solving "
+            "subtask.",
+            taxonomy=TaxonomyInfo(
+                task="next-word prediction", what="exam", when="2010s", who="n/a", language="Chinese"
+            ),
+            main_metric="cleva_math_result_match",
+            main_split="test",
+        )
+
 
 class CLEVALanguageModelingScenario(CLEVAScenario):
     """
@@ -1547,9 +2000,22 @@ class CLEVALanguageModelingScenario(CLEVAScenario):
         )
         return instance
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_language_modeling",
+            display_name="CLEVA (Chinese) language modeling",
+            description="Scenario for measuring language model performance across various domains "
+            "(wikipedia and news).",
+            taxonomy=TaxonomyInfo(
+                task="language modeling", what="Wikipedia and news", when="2010s", who="n/a", language="Chinese"
+            ),
+            main_metric="bits_per_byte",
+            main_split="test",
+        )
+
 
 class CLEVACodeSynthesisScenario(CLEVAScenario):
-    """
+    r"""
     The code synthesis task of CLEVA benchmark.
 
     An example is:
@@ -1580,6 +2046,19 @@ class CLEVACodeSynthesisScenario(CLEVAScenario):
         instance = self.converter.transform_code(row, self.prompt_template, split)
         return instance
 
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="cleva_code_synthesis",
+            display_name="CLEVA (Chinese) code synthesis",
+            description="Scenario for measuring functional correctness for synthesizing programs from "
+            "Chinese docstrings.",
+            taxonomy=TaxonomyInfo(
+                task="next-word prediction", what="n/a", when="2023", who="n/a", language="synthetic"
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )
+
 
 class CLEVAKeyphraseExtractionScenario(CLEVAScenario):
     """