crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/koala_scenario.py

@@ -2,8 +2,9 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Input, TEST_SPLIT
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TEST_SPLIT, ScenarioMetadata
 
 
 class KoalaScenario(Scenario):
@@ -39,3 +40,22 @@ class KoalaScenario(Scenario):
             )
             instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="koala",
+            display_name="Koala test dataset",
+            short_display_name="Koala test dataset",
+            description="The test dataset from the [Koala "
+            "paper](https://bair.berkeley.edu/blog/2023/04/03/koala/) for evaluating "
+            "instruction-following models.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Instructions for LLMs",
+                when="Before 2023",
+                who="Web users",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/kpi_edgar_scenario.py

@@ -0,0 +1,172 @@
+import os
+from typing import List, Dict
+import json
+import re
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
+
+
+class KPIEDGARScenario(Scenario):
+    """A financial named entity recognition (NER) scenario based on KPI-EDGAR (T. Deußer et al., 2022).
+
+    This scenario has been modified from the paper. The original paper has 12 entity types and requires the model
+    to extract pairs of related entities. This scenario only use four named entity types (kpi, cy, py, py1) and only
+    requires the model to extract individual entities.
+
+    Paper:
+    T. Deußer et al.,
+    “KPI-EDGAR: A Novel Dataset and Accompanying Metric for Relation Extraction from Financial Documents.” 2022.
+    https://arxiv.org/abs/2210.09163
+
+    Prompt format:
+
+    ```
+    Context: {Sentence}
+    Task: Extract key performance indicators (KPIs) and values from the above text. Also, specify one of the following categories to each of the extracted KPIs and values in brackets.
+    kpi: Key Performance Indicators expressible in numerical and monetary value, cy: Current Year monetary value, py: Prior Year monetary value, py1: Two Year Past Value.
+    Answer:
+    ```
+
+    Example input:
+
+    ```
+    Context: The following table summarizes our total share-based compensation expense and excess tax benefits recognized : As of December 28 , 2019 , there was $ 284 million of total unrecognized compensation cost related to nonvested share-based compensation grants .
+    Task: Extract key performance indicators (KPIs) and values from the above text. Also, specify one of the following categories to each of the extracted KPIs and values in brackets.
+    kpi: Key Performance Indicators expressible in numerical and monetary value, cy: Current Year monetary value, py: Prior Year monetary value, py1: Two Year Past Value.
+    Answer:
+    ```
+
+    Example reference:
+    ```
+    284 [cy], total unrecognized compensation cost [kpi]
+    ```"""  # noqa: E501
+
+    name = "kpi_edgar"
+    description = "Named Entity Recognition from financial documents."
+    tags = ["named_entity_recognition", "finance"]
+
+    TAG_DICT = {
+        "kpi": "Key Performance Indicators expressible in numerical and monetary value",
+        "cy": "Current Year monetary value",
+        "py": "Prior Year monetary value",
+        "py1": "Two Year Past Value",
+    }
+    TAG_DESCRIPTIONS = ", ".join(["%s: %s" % (key, val) for (key, val) in TAG_DICT.items()]) + "."
+    TAG_PAREN_RE = (r"\[", r"\]")
+    TAG_PAREN = tuple((e.strip("\\") for e in TAG_PAREN_RE))
+    TAG_PAREN_ESC = ("(", ")")
+    DATASET_SPLIT_TO_HELM_SPLIT = {"train": TRAIN_SPLIT, "valid": VALID_SPLIT, "test": TEST_SPLIT}
+    JSON_URL = "https://raw.githubusercontent.com/tobideusser/kpi-edgar/2ec7084dcd55b4979bbe288d4aa1e962c685c9ab/data/kpi_edgar.json"  # noqa: E501
+    JSON_FILENAME = "kpi_edgar.json"
+
+    @staticmethod
+    def get_sentences(dataset: List[Dict]) -> List[Dict]:
+        return [
+            sentence
+            for document in dataset
+            for segment in document["segments"]
+            for sentence in segment["sentences"] or []
+        ]
+
+    @staticmethod
+    def escape_parenthesis(text: str) -> str:
+        tmp0 = re.sub(KPIEDGARScenario.TAG_PAREN_RE[0], KPIEDGARScenario.TAG_PAREN_ESC[0], text)
+        tmp1 = re.sub(KPIEDGARScenario.TAG_PAREN_RE[1], KPIEDGARScenario.TAG_PAREN_ESC[1], tmp0)
+        return tmp1
+
+    @staticmethod
+    def get_output_text(
+        words: List[str],
+        annotations: List[Dict],
+    ) -> str:
+        # def get_entity_for_annotation(words: List[str], annotation: Dict) -> str
+        entities: List[str] = []
+        for annotation in annotations:
+            annotation_type = annotation["type_"]
+            if annotation_type not in KPIEDGARScenario.TAG_DICT:
+                continue
+            start_idx = annotation["start"]
+            end_idx = annotation["end"]
+            annotated_words = words[start_idx:end_idx]
+            phrase = KPIEDGARScenario.escape_parenthesis(" ".join(annotated_words))
+            entities.append(
+                "%s %s%s%s" % (phrase, KPIEDGARScenario.TAG_PAREN[0], annotation_type, KPIEDGARScenario.TAG_PAREN[1])
+            )
+
+        return ", ".join(entities)
+
+    @staticmethod
+    def sentences_to_instances(sentences: List[Dict]) -> List[Instance]:
+        instances: List[Instance] = []
+        for sentence in sentences:
+            dataset_split: str = sentence["split_type"]
+            if dataset_split is None:
+                continue
+            split = KPIEDGARScenario.DATASET_SPLIT_TO_HELM_SPLIT[dataset_split]
+
+            words: List[str] = [word_dict["value"] for word_dict in sentence["words"]]
+            passage = KPIEDGARScenario.escape_parenthesis(" ".join(words))
+            input_text = (
+                "Context: %s\n"
+                "Task: Extract key performance indicators (KPIs) and values from the above text. Also, specify one of the following categories to each of the extracted KPIs and values in brackets.\n"  # noqa: E501
+                "%s" % (passage, KPIEDGARScenario.TAG_DESCRIPTIONS)
+            )
+
+            annotations = sentence["entities_anno"]
+            output_text = KPIEDGARScenario.get_output_text(words, annotations)
+            if not output_text:
+                continue
+
+            instances.append(
+                Instance(
+                    input=Input(text=input_text),
+                    references=[Reference(Output(text=output_text), tags=[CORRECT_TAG])],
+                    split=split,
+                )
+            )
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        data_path = os.path.join(output_path, "data")
+        ensure_directory_exists(data_path)
+
+        base_url = self.JSON_URL
+        dataset_file_name = self.JSON_FILENAME
+        target_path = os.path.join(data_path, dataset_file_name)
+        ensure_file_downloaded(source_url=base_url, target_path=target_path)
+
+        with open(target_path, "r") as f:
+            raw_dataset = json.load(f)
+        return KPIEDGARScenario.sentences_to_instances(KPIEDGARScenario.get_sentences(raw_dataset))
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="kpi_edgar",
+            display_name="KPI-EDGAR Financial Documents (Named Entity Recognition)",
+            short_display_name=None,
+            description="A named entity recognition beenchmark based on the paper KPI-EDGAR - A Novel "
+            "Dataset and Accompanying Metric for Relation Extraction from Financial "
+            "Documents [(Deußer et al., 2022)](https://arxiv.org/pdf/2210.09163.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="named entity recognition",
+                what="financial reports",
+                when="before 2022",
+                who="financial experts",
+                language="English",
+            ),
+            main_metric="adjusted_macro_f1_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/legal_contract_summarization_scenario.py

@@ -0,0 +1,149 @@
+import os
+import pandas as pd
+import json
+import re
+
+from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Input,
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Output,
+    ScenarioMetadata,
+)
+
+
+class LegalContractSummarizationScenario(Scenario):
+    """Legal Contract Summarization
+
+    A legal contract summarization benchmark based on the paper
+    Plain English Summarization of Contracts (Manor & Li, NAACL 2019),
+    which presented a dataset of legal text snippets paired with summaries
+    written in plain English.
+
+    @inproceedings{manor-li-2019-plain,
+        title = "Plain {E}nglish Summarization of Contracts",
+        author = "Manor, Laura  and
+        Li, Junyi Jessy",
+        editor = "Aletras, Nikolaos  and
+        Ash, Elliott  and
+        Barrett, Leslie  and
+        Chen, Daniel  and
+        Meyers, Adam  and
+        Preotiuc-Pietro, Daniel  and
+        Rosenberg, David  and
+        Stent, Amanda",
+        booktitle = "Proceedings of the Natural Legal Language Processing Workshop 2019",
+        month = jun,
+        year = "2019",
+        address = "Minneapolis, Minnesota",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/W19-2201",
+        doi = "10.18653/v1/W19-2201",
+        pages = "1--11",
+        abstract = "Unilateral legal contracts, such as terms of service, play a substantial role in modern digital life. However, few read these documents before accepting the terms within, as they are too long and the language too complicated. We propose the task of summarizing such legal documents in plain English, which would enable users to have a better understanding of the terms they are accepting. We propose an initial dataset of legal text snippets paired with summaries written in plain English. We verify the quality of these summaries manually, and show that they involve heavy abstraction, compression, and simplification. Initial experiments show that unsupervised extractive summarization methods do not perform well on this task due to the level of abstraction and style differences. We conclude with a call for resource and technique development for simplification and style transfer for legal language.",
+    }
+    """  # noqa: E501
+
+    TRAIN_RATIO: float = 0.2
+    ARTICLE_COLUMN_NAME = "original_text"
+    SUMMARY_COLUMN_NAME = "reference_summary"
+    ID_COLUMN_NAME = "uid"
+
+    name = "legal_contract_summarization"
+    description = (
+        "Plain English Summarization of Contracts [(Manor et al., 2019)](https://aclanthology.org/W19-2201.pdf)."
+    )
+    tags = ["summarization", "legal"]
+
+    def __init__(self):
+        """
+        Initializes the scenario.
+
+        """
+        super().__init__()
+
+    @staticmethod
+    def _clean(text: str) -> str:
+        return re.sub(r"\s+", " ", text)
+
+    def _load_dataset(self, output_path: str):
+        data_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(data_dir)
+
+        source_url = "https://raw.githubusercontent.com/lauramanor/legal_summarization/master/all_v1.json"
+        source_file = os.path.basename(source_url)
+        target_path = os.path.join(data_dir, source_file)
+        ensure_file_downloaded(
+            source_url=source_url,
+            target_path=target_path,
+        )
+
+        target_df = pd.DataFrame()
+        with open(target_path) as f:
+            json_data = json.load(f)
+            target_df = pd.DataFrame.from_records(list(json_data.values()))
+            target_df = target_df.dropna(
+                subset=[
+                    LegalContractSummarizationScenario.ARTICLE_COLUMN_NAME,
+                    LegalContractSummarizationScenario.SUMMARY_COLUMN_NAME,
+                    LegalContractSummarizationScenario.ID_COLUMN_NAME,
+                ]
+            )
+            # Split randomly (works better than split by order)
+            train_df = target_df.sample(frac=LegalContractSummarizationScenario.TRAIN_RATIO, random_state=0)
+            test_df = target_df.drop(train_df.index).sample(frac=1, random_state=0)
+
+        return {TRAIN_SPLIT: train_df, TEST_SPLIT: test_df}
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset = self._load_dataset(output_path)
+
+        instances: List[Instance] = []
+
+        for split, split_data in dataset.items():
+            for example in split_data.itertuples():
+                id = getattr(example, LegalContractSummarizationScenario.ID_COLUMN_NAME)
+                article = LegalContractSummarizationScenario._clean(
+                    getattr(example, LegalContractSummarizationScenario.ARTICLE_COLUMN_NAME)
+                )
+                summary = LegalContractSummarizationScenario._clean(
+                    getattr(example, LegalContractSummarizationScenario.SUMMARY_COLUMN_NAME)
+                )
+                input = Input(
+                    text=article,
+                )
+                output = Output(text=summary)
+                instance = Instance(
+                    id=id,
+                    input=input,
+                    references=[Reference(output=output, tags=[CORRECT_TAG])],
+                    split=split,
+                )
+                instances.append(instance)
+
+        return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="legal_contract_summarization",
+            display_name="Legal Contract Summarization",
+            short_display_name=None,
+            description="Plain English Summarization of Contracts [(Manor et al., "
+            "2019)](https://aclanthology.org/W19-2201.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="summarization",
+                what="legal contracts (e.g. terms of service, license agreements)",
+                when="before 2019",
+                who="lawyers",
+                language="English",
+            ),
+            main_metric="rouge_l",
+            main_split="test",
+        )

helm/benchmark/scenarios/legal_opinion_sentiment_classification_scenario.py

@@ -0,0 +1,77 @@
+import os
+from typing import List
+
+import pandas as pd
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.general import ensure_file_downloaded, ensure_directory_exists
+
+
+class LegalOpinionSentimentClassificationScenario(Scenario):
+    """
+    A legal opinion sentiment classification task based on the paper
+    Effective Approach to Develop a Sentiment Annotator For Legal Domain in a Low Resource Setting
+    [(Ratnayaka et al., 2020)](https://arxiv.org/pdf/2011.00318.pdf).
+
+    Example prompt:
+    Classify the sentences into one of the 3 sentiment categories. Possible labels: positive, neutral, negative.
+    {Sentence}
+    Label: {positive/neutral/negative}
+
+    """
+
+    # Names of the tasks we support
+
+    name = "legal_opinion"
+    description = "Predicting the sentiment of the legal text in the positive, negative, or neutral."
+    tags = ["classification", "sentiment analysis", "legal"]
+
+    SENTIMENT_CLASSES = ["positive", "negative", "neutral"]
+    SPLIT_TO_URL = {
+        TRAIN_SPLIT: "https://osf.io/download/hfn62/",
+        TEST_SPLIT: "https://osf.io/download/q4adh/",
+    }
+
+    def create_instances(self, df: pd.DataFrame, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        assert split in [TRAIN_SPLIT, TEST_SPLIT]
+        if split == TRAIN_SPLIT:
+            phrase_column_name = "Phrase"
+            label_column_name = "Label"
+        else:
+            phrase_column_name = "sentence"
+            label_column_name = "label"
+        for row in df.itertuples():
+            phrase = getattr(row, phrase_column_name)
+            label_index = int(getattr(row, label_column_name))
+            label = LegalOpinionSentimentClassificationScenario.SENTIMENT_CLASSES[label_index]
+            instance = Instance(
+                input=Input(text=phrase), references=[Reference(Output(text=label), tags=[CORRECT_TAG])], split=split
+            )
+            instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        self.data_dir = os.path.join(output_path, "data")
+        data_dir = self.data_dir
+        ensure_directory_exists(data_dir)
+        instances: List[Instance] = []
+        for split, url in LegalOpinionSentimentClassificationScenario.SPLIT_TO_URL.items():
+            file_name = f"{split.lower()}.xlsx"
+            file_path = os.path.join(data_dir, file_name)
+            ensure_file_downloaded(
+                source_url=url,
+                target_path=os.path.join(data_dir, file_name),
+            )
+            df = pd.read_excel(file_path)
+            instances.extend(self.create_instances(df, split))
+        return instances

helm/benchmark/scenarios/legal_summarization_scenario.py

@@ -5,7 +5,19 @@ from typing import List, Optional, Any
 import datasets
 from datasets import load_dataset
 
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 _ALL_LANGUAGES = {
     "bulgarian": "bg",
@@ -195,3 +207,51 @@ class LegalSummarizationScenario(Scenario):
                 )
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        if self.dataset_name == "BillSum":
+            return ScenarioMetadata(
+                name="billsum_legal_summarization",
+                display_name="BillSum",
+                description="The BillSum benchmark for legal text summarization ([Kornilova & Eidelmann, "
+                "2020](https://aclanthology.org/D19-5406/)).",
+                taxonomy=TaxonomyInfo(
+                    task="summarization", what="legal text from US bills", when=None, who="lawyers", language="English"
+                ),
+                main_metric="rouge_2",
+                main_split="test",
+            )
+        elif self.dataset_name == "MultiLexSum":
+            return ScenarioMetadata(
+                name="multilexsum_legal_summarization",
+                display_name="MultiLexSum",
+                description="The MultiLexSum benchmark for legal text summarization ([Shen et al., "
+                "2022](https://arxiv.org/abs/2206.10883)).",
+                taxonomy=TaxonomyInfo(
+                    task="summarization",
+                    what="legal text from US civil rights lawsuits",
+                    when=None,
+                    who="lawyers",
+                    language="English",
+                ),
+                main_metric="rouge_2",
+                main_split="test",
+            )
+        elif self.dataset_name == "EurLexSum":
+            return ScenarioMetadata(
+                name="eurlexsum_legal_summarization",
+                display_name="EurLexSum",
+                description="The EurLexSum benchmark for legal text summarization ([Aumiller et al., "
+                "2022](https://arxiv.org/abs/2210.13448)).",
+                taxonomy=TaxonomyInfo(
+                    task="summarization",
+                    what="legal text from EU legislation",
+                    when="1960 - 2020",
+                    who="lawyers",
+                    language="English",
+                ),
+                main_metric="rouge_2",
+                main_split="test",
+            )
+        else:
+            raise Exception(f"Unknown dataset {self.dataset_name}")

helm/benchmark/scenarios/legal_support_scenario.py

@@ -2,8 +2,20 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
-from .scenario import Scenario, Instance, Reference, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, CORRECT_TAG, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 
 class LegalSupportScenario(Scenario):
@@ -92,3 +104,14 @@ class LegalSupportScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="legal_support",
+            display_name="LegalSupport",
+            description="Scenario introduced in this work to measure fine-grained legal reasoning "
+            "through reverse entailment.",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/legalbench_scenario.py

@@ -5,8 +5,19 @@ import datasets
 from pathlib import Path
 from typing import List, Dict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
-from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TRAIN_SPLIT, TEST_SPLIT, Input, Output
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 PROMPT_SETTINGS_URL = "https://raw.githubusercontent.com/HazyResearch/legalbench/main/helm_prompt_settings.jsonl"
 
@@ -96,8 +107,22 @@ class LegalBenchScenario(Scenario):
 
         # Download data from Huggingface. LegalBench provides splits for samples to
         # be used for prompt construction and for testing.
-        train_dataset = datasets.load_dataset("nguha/legalbench", self.subset, cache_dir=cache_dir, split="train")
-        test_dataset = datasets.load_dataset("nguha/legalbench", self.subset, cache_dir=cache_dir, split="test")
+        train_dataset = datasets.load_dataset(
+            "nguha/legalbench",
+            self.subset,
+            trust_remote_code=True,
+            cache_dir=cache_dir,
+            split="train",
+            revision="e042ea68c19df12b737fe768572f22ead61e8e37",
+        )
+        test_dataset = datasets.load_dataset(
+            "nguha/legalbench",
+            self.subset,
+            trust_remote_code=True,
+            cache_dir=cache_dir,
+            split="test",
+            revision="e042ea68c19df12b737fe768572f22ead61e8e37",
+        )
         assert isinstance(train_dataset, datasets.Dataset)
         assert isinstance(test_dataset, datasets.Dataset)
 
@@ -121,3 +146,20 @@ class LegalBenchScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="LegalBench",
+            description="LegalBench is a large collaboratively constructed benchmark of legal reasoning "
+            "tasks [(Guha et al, 2023)](https://arxiv.org/pdf/2308.11462.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="public legal and admininstrative documents, manually " "constructed questions",
+                when="before 2023",
+                who="lawyers",
+                language="English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/lex_glue_scenario.py

@@ -5,8 +5,19 @@ from typing import List, Any
 import datasets
 from datasets import load_dataset
 
-from .lextreme_scenario import TaskType
-from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, Input, Output
+from helm.benchmark.scenarios.lextreme_scenario import TaskType
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    Input,
+    Output,
+    ScenarioMetadata,
+)
 
 ECTHR_A = "ecthr_a"
 ECTHR_B = "ecthr_b"
@@ -251,3 +262,13 @@ class LexGLUEScenario(Scenario):
         for subset in self.subsets:
             instances.extend(self.get_instances_for_subset(subset, output_path))
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="lex_glue",
+            display_name="LexGLUE",
+            description="A Benchmark Dataset for Legal Language Understanding in English",
+            taxonomy=None,
+            main_metric="classification_macro_f1",
+            main_split="test",
+        )

helm/benchmark/scenarios/lextreme_scenario.py

@@ -6,7 +6,18 @@ from typing import List, Any
 import datasets
 from datasets import load_dataset
 
-from .scenario import Scenario, Instance, Reference, CORRECT_TAG, TRAIN_SPLIT, VALID_SPLIT, TEST_SPLIT, Output, Input
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    CORRECT_TAG,
+    TRAIN_SPLIT,
+    VALID_SPLIT,
+    TEST_SPLIT,
+    Output,
+    Input,
+    ScenarioMetadata,
+)
 
 
 class TaskType:
@@ -456,3 +467,13 @@ class LEXTREMEScenario(Scenario):
         for subset in self.subsets:
             instances.extend(self.get_instances_for_subset(subset, output_path))
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="lextreme",
+            display_name="LEXTREME",
+            description="A Multilingual Legal Benchmark for Natural Language Understanding",
+            taxonomy=None,
+            main_metric="classification_macro_f1",
+            main_split="test",
+        )