crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/legal_support_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -102,3 +104,14 @@ class LegalSupportScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="legal_support",
+            display_name="LegalSupport",
+            description="Scenario introduced in this work to measure fine-grained legal reasoning "
+            "through reverse entailment.",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/legalbench_scenario.py

@@ -5,6 +5,7 @@ import datasets
 from pathlib import Path
 from typing import List, Dict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -15,6 +16,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 PROMPT_SETTINGS_URL = "https://raw.githubusercontent.com/HazyResearch/legalbench/main/helm_prompt_settings.jsonl"
@@ -144,3 +146,20 @@ class LegalBenchScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name=self.name,
+            display_name="LegalBench",
+            description="LegalBench is a large collaboratively constructed benchmark of legal reasoning "
+            "tasks [(Guha et al, 2023)](https://arxiv.org/pdf/2308.11462.pdf).",
+            taxonomy=TaxonomyInfo(
+                task="multiple-choice question answering",
+                what="public legal and admininstrative documents, manually " "constructed questions",
+                when="before 2023",
+                who="lawyers",
+                language="English",
+            ),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/lex_glue_scenario.py

@@ -16,6 +16,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 ECTHR_A = "ecthr_a"
@@ -261,3 +262,13 @@ class LexGLUEScenario(Scenario):
         for subset in self.subsets:
             instances.extend(self.get_instances_for_subset(subset, output_path))
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="lex_glue",
+            display_name="LexGLUE",
+            description="A Benchmark Dataset for Legal Language Understanding in English",
+            taxonomy=None,
+            main_metric="classification_macro_f1",
+            main_split="test",
+        )

helm/benchmark/scenarios/lextreme_scenario.py

@@ -16,6 +16,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Output,
     Input,
+    ScenarioMetadata,
 )
 
 
@@ -466,3 +467,13 @@ class LEXTREMEScenario(Scenario):
         for subset in self.subsets:
             instances.extend(self.get_instances_for_subset(subset, output_path))
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="lextreme",
+            display_name="LEXTREME",
+            description="A Multilingual Legal Benchmark for Natural Language Understanding",
+            taxonomy=None,
+            main_metric="classification_macro_f1",
+            main_split="test",
+        )

helm/benchmark/scenarios/lmkt_scenarios.py

@@ -0,0 +1,288 @@
+"""Cultural alignment evaluation scenario based on Vietnam World Values Survey responses."""
+
+import os
+import json
+import random
+from typing import List
+from datasets import load_dataset
+from huggingface_hub import snapshot_download
+
+from helm.common.hierarchical_logger import hlog, hwarn
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+SUPPORTED_LANGUAGES = ["en", "vi"]
+
+
+class CulturalValueUnderstandingWVSScenario(Scenario):
+    """Cultural values understanding evaluation based on Vietnam World Values Survey responses."""
+
+    name = "cultural_value_understanding_wvs"
+    description = "Evaluates model understanding of cultural values from WVS Wave 7"
+    tags = ["cultural_value_understanding"]
+
+    def __init__(
+        self,
+        language: str,
+        personas_filename: str = "vn_personas_with_filtered_responses.json",
+        questions_filename: str = "wvs_questions_dump.en.json",
+        repo_id: str = "stair-lab/cultural-alignment-dataset",
+        num_personas: int = 10,
+        num_question_variants: int = 2,
+        include_few_shot_examples: bool = False,
+    ):
+        super().__init__()
+        self.language = language
+        self.personas_filename = personas_filename
+        self.questions_filename = questions_filename
+        self.repo_id = repo_id
+        self.num_personas = 10 if isinstance(num_personas, str) else num_personas
+        self.num_question_variants = min(num_question_variants, 4)
+        self.include_few_shot_examples = include_few_shot_examples
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_few_shot_examples(self) -> str:
+        """Return a few-shot learning examples formatted as a string."""
+        examples = """Here are some examples:
+
+Example 1:
+You are a 35 year old male from Vietnam. You have higher education and consider yourself middle class. You are married.
+
+How important is family in your life?
+
+1. Very important
+2. Rather important
+3. Not very important
+4. Not at all important
+
+Please select the option number that best represents your view.
+1
+
+Example 2:
+You are a 28 year old female from Vietnam. You have middle education and consider yourself working class. You are single.
+
+Would you say that most people can be trusted or that you need to be very careful in dealing with people?
+
+1. Most people can be trusted
+2. Need to be very careful
+
+Please select the option number that best represents your view.
+2
+
+Example 3:
+You are a 50 year old male from Vietnam. You have lower education and consider yourself lower class. You are married.
+
+Do you think that homosexuality is justifiable?
+
+1. Never justifiable
+2. Rarely justifiable
+3. Sometimes justifiable
+4. Always justifiable
+
+Please select the option number that best represents your view.
+1
+
+Now answer the following question:
+"""  # noqa: E501
+        return examples
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Generate test instances from Vietnam personas and WVS questions."""
+        instances: List[Instance] = []
+
+        # Download files from Hugging Face Hub
+        repo_local_path = snapshot_download(
+            repo_id=self.repo_id, repo_type="dataset", revision="fe54b6f5d75cfca5377707cd7199e39f517e3a1f"
+        )
+
+        # Load the downloaded files
+        with open(os.path.join(repo_local_path, self.personas_filename), "r", encoding="utf-8") as f:
+            personas = json.load(f)
+
+        with open(os.path.join(repo_local_path, self.questions_filename), "r", encoding="utf-8") as f:
+            questions = json.load(f)
+
+        # Get few-shot examples
+        few_shot_examples = self.get_few_shot_examples() if self.include_few_shot_examples else ""
+
+        # Sample personas
+        sampled_personas = random.sample(personas, min(self.num_personas, len(personas)))
+
+        # Create instances for each persona and question
+        for persona in sampled_personas:
+            # Get demographic info for persona description
+            persona_desc = (
+                f"You are a {persona.get('age', 'adult')} year old {persona.get('sex', 'person')} from Vietnam. "
+            )
+            persona_desc += f"You have {persona.get('education', 'some')} education and consider yourself {persona.get('social_class', 'middle class')}. "  # noqa: E501
+            persona_desc += f"You are {persona.get('marital_status', 'single')}."
+
+            # Process each question this persona answered
+            for qid, human_response in persona.get("responses", {}).items():
+                # Skip if no human response or if it's 0 (which might be a "Don't know" response)
+                if human_response is None:
+                    continue
+
+                # Convert human_response to int (if possible)
+                try:
+                    human_response_int = int(human_response)
+                except (ValueError, TypeError):
+                    # Skip if human_response can't be converted to int
+                    continue
+
+                # Get question info
+                question_data = questions.get(qid, {})
+                if not question_data:
+                    continue
+
+                # Get options directly from question data
+                q_options = question_data.get("options", [])
+                if not q_options:
+                    continue
+
+                # Skip if human_response is out of range
+                if human_response_int < 0 or human_response_int > len(q_options):
+                    continue
+
+                # Special handling for "Don't know" or zero responses
+                if human_response_int == 0:
+                    # Some questions might encode "Don't know" as 0
+                    # Skip for now, or you could add special handling
+                    continue
+
+                # Use the predefined question variations
+                question_variants = question_data.get("questions", [])
+                if not question_variants:
+                    question_variants = [f"Question {qid}: {question_data.get('description', '')}"]
+
+                # Use the specified number of variants
+                variants_to_use = min(self.num_question_variants, len(question_variants))
+                selected_variants = question_variants[:variants_to_use]
+
+                # Create instances for each selected question variant
+                for q_text in selected_variants:
+                    # Format the prompt with or without few-shot examples
+                    if self.include_few_shot_examples:
+                        prompt = f"{few_shot_examples}{persona_desc}\n\n{q_text}\n\n"
+                    else:
+                        prompt = f"{persona_desc}\n\n{q_text}\n\n"
+
+                    # Add options from question data - with numbers, not letters
+                    for i, opt in enumerate(q_options, 1):
+                        prompt += f"{i}. {opt}\n"
+
+                    prompt += "\nPlease select the option number that best represents your view. Return only the option number. Do not return anything else."  # noqa: E501
+
+                    # Create a reference with just the human response number
+                    # We don't create multiple references, just use the actual human response
+                    reference = Reference(Output(text=str(human_response_int)), tags=[CORRECT_TAG])
+
+                    # Create the instance
+                    instance = Instance(
+                        input=Input(text=prompt),
+                        references=[reference],
+                        split=TEST_SPLIT,
+                    )
+                    instances.append(instance)
+
+        if not instances:
+            hwarn("No valid instances were created. Check the input data and parameters.")
+        else:
+            hlog(f"Created {len(instances)} instances for cultural value understanding scenario.")
+
+        return instances
+
+
+class SocialNormApplicationNormADScenario(Scenario):
+    """Social norms application evaluation based on Vietnam World Values Survey responses."""
+
+    name = "social_norm_application_normad"
+    description = "Evaluates model understanding of social norms from WVS Wave 7"
+    tags = ["social_norm_application"]
+
+    def __init__(
+        self,
+        language: str,
+    ):
+        super().__init__()
+        self.language = language
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset = load_dataset(
+            "akhilayerukola/NormAd", split="train", revision="311049f5610e932792672e4e13e7df62b6290336"
+        )
+
+        instances: List[Instance] = []
+
+        for item in dataset:
+            input_text = (
+                f"Assumming that below norms and story are from {item['Country'].capitalize()}:\n"
+                f"{item['Background']}\n\n{item['Story']}\n"
+                "Return one of the following options: 'yes', 'no', or 'neutral'. Do not return anything else.\n\n"
+            )
+
+            instance = Instance(
+                input=Input(text=input_text),
+                references=[Reference(Output(text=item["Gold Label"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances
+
+
+class SocialNormExplanationNormADScenario(Scenario):
+    """Social norms explanation evaluation based on Vietnam World Values Survey responses."""
+
+    name = "social_norm_explanation_normad"
+    description = "Evaluates model understanding of social norms from WVS Wave 7"
+    tags = ["social_norm_explanation"]
+
+    def __init__(
+        self,
+        language: str,
+    ):
+        super().__init__()
+        self.language = language
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset = load_dataset(
+            "akhilayerukola/NormAd", split="train", revision="311049f5610e932792672e4e13e7df62b6290336"
+        )
+
+        instances: List[Instance] = []
+
+        for item in dataset:
+            if not item["Explanation"]:
+                continue
+
+            input_text = (
+                f"Assumming that below norms and story are from {item['Country'].capitalize()}:\n"
+                f"{item['Background']}\n\n{item['Story']}\n"
+                f"The answer is {item['Gold Label']}. "
+                "Briefly explain the reasoning behind this answer in one or two sentences.\n\n"
+            )
+
+            instance = Instance(
+                input=Input(text=input_text),
+                references=[Reference(Output(text=item["Explanation"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/lsat_qa_scenario.py

@@ -2,6 +2,7 @@ import os
 import json
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -13,6 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -143,3 +145,15 @@ class LSATScenario(Scenario):
                             instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="lsat_qa",
+            display_name="LSAT",
+            description="The LSAT benchmark for measuring analytical reasoning on the Law School "
+            "Admission Test (LSAT; [Zhong et al., "
+            "2021](https://arxiv.org/pdf/2104.06598.pdf)).",
+            taxonomy=TaxonomyInfo(task="?", what="n/a", when="n/a", who="n/a", language="synthetic"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/madinah_qa_scenario.py

@@ -0,0 +1,73 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MadinahQAScenario(Scenario):
+    """MadinahQA Scenario"""
+
+    name = "madinah_qa"
+    description = "Arabic language competency benchmark"
+    tags = ["language", "multiple_choice"]
+
+    OPTIONS = ["A", "B", "C", "D"]
+    HF_SPLIT_TO_HELM_SPLIT = {"dev": TRAIN_SPLIT, "test": TEST_SPLIT}
+    SUBSETS = ["Arabic Language (General)", "Arabic Language (Grammar)"]
+
+    def __init__(self, subset: str):
+        super().__init__()
+        subset = subset.replace("_", " ")
+        if subset not in self.SUBSETS:
+            raise Exception(f"Unknown subset: {subset}")
+        self.subset = subset
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        instances: List[Instance] = []
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "MBZUAI/MadinahQA",
+            self.subset,
+            revision="62e7c86ac5c07245a5a952722691d77ddb41f695",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            for row_index, row in enumerate(dataset):
+                input = Input(text=row["Question"])
+                references: List[Reference] = []
+                correct_option_index = ord(row["Answer Key"]) - ord("A") + 1
+                for option_index in range(1, 6):
+                    column_name = f"Option {option_index}"
+                    if not row[column_name]:
+                        continue
+                    references.append(
+                        Reference(
+                            output=Output(text=row[column_name]),
+                            tags=[CORRECT_TAG] if option_index == correct_option_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=f"id{row_index}",
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/math_scenario.py

@@ -4,6 +4,7 @@ import typing
 from typing import Dict, List, Optional
 from datasets import load_dataset, DatasetDict
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_directory_exists
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -14,17 +15,19 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
 def remove_boxed(string: str) -> Optional[str]:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
-    Extract the text within a \\boxed{...} environment.
+    Extract the text within a \boxed{...} environment.
 
     Example:
-    >>> remove_boxed(\\boxed{\\frac{2}{3}})
-    \\frac{2}{3}
+        >>> from helm.benchmark.scenarios.math_scenario import *  # NOQA
+        >>> remove_boxed(r'\boxed{\frac{2}{3}}')
+        '\\frac{2}{3}'
     """
     left = "\\boxed{"
     try:
@@ -68,17 +71,17 @@ def last_boxed_only_string(string: str) -> Optional[str]:
 
 
 def _fix_fracs(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat fractions.
 
     Examples:
-    >>> _fix_fracs("\\frac1b")
-    \frac{1}{b}
-    >>> _fix_fracs("\\frac12")
-    \frac{1}{2}
-    >>> _fix_fracs("\\frac1{72}")
-    \frac{1}{72}
+        >>> _fix_fracs(r"\frac1b")
+        '\\frac{1}{b}'
+        >>> _fix_fracs(r"\frac12")
+        '\\frac{1}{2}'
+        >>> _fix_fracs(r"\frac1{72}")
+        '\\frac{1}{72}'
     """
     substrs = string.split("\\frac")
     new_str = substrs[0]
@@ -112,13 +115,13 @@ def _fix_fracs(string: str) -> str:
 
 
 def _fix_a_slash_b(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat fractions formatted as a/b to \\frac{a}{b}.
 
     Example:
-    >>> _fix_a_slash_b("2/3")
-    \frac{2}{3}
+        >>> _fix_a_slash_b(r"2/3")
+        '\\frac{2}{3}'
     """
     if len(string.split("/")) != 2:
         return string
@@ -149,13 +152,13 @@ def _remove_right_units(string: str) -> str:
 
 
 def _fix_sqrt(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat square roots.
 
     Example:
-    >>> _fix_sqrt("\\sqrt3")
-    \sqrt{3}
+        >>> _fix_sqrt("\\sqrt3")
+        '\\sqrt{3}'
     """
     if "\\sqrt" not in string:
         return string
@@ -210,7 +213,7 @@ def _strip_string(string: str) -> str:
 
     # remove percentage
     string = string.replace("\\%", "")
-    string = string.replace("\%", "")
+    string = string.replace(r"\%", "")
 
     # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
     string = string.replace(" .", " 0.")
@@ -391,13 +394,13 @@ class MATHScenario(Scenario):
         for split, split_name in zip([TRAIN_SPLIT, TEST_SPLIT], ["train", "test"]):
             if split == TRAIN_SPLIT and self.use_official_examples:
                 train_instances = [
-                    ("What is $\left(\\frac{7}{8}\\right)^3 \cdot \left(\\frac{7}{8}\\right)^{-3}$?", "1"),
+                    ("What is $\\left(\\frac{7}{8}\\right)^3 \\cdot \\left(\\frac{7}{8}\\right)^{-3}$?", "1"),
                     (
                         "In how many ways can 4 books be selected from a shelf of 6 books"
                         + " if the order in which the books are selected does not matter?",
                         "15",
                     ),
-                    ("Find the distance between the points $(2,1,-4)$ and $(5,8,-3).$", "\sqrt{59}"),
+                    ("Find the distance between the points $(2,1,-4)$ and $(5,8,-3).$", "\\sqrt{59}"),
                     (
                         "The faces of an octahedral die are labeled with digits $1$ through $8$."
                         + " What is the probability, expressed as a common fraction,"
@@ -449,3 +452,34 @@ class MATHScenario(Scenario):
                 instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        taxonomy = TaxonomyInfo(
+            task="numeric answer question answering",
+            what="math competitions (AMC, AIME, etc.)",
+            when="before 2021",
+            who="problem setters",
+            language="synthetic",
+        )
+        if self.use_chain_of_thought:
+            return ScenarioMetadata(
+                name="math_chain_of_thought",
+                display_name="MATH",
+                description="The MATH benchmark for measuring mathematical problem solving on competition "
+                "math problems with chain-of-thought style reasoning [(Hendrycks et al., "
+                "2021)](https://arxiv.org/pdf/2103.03874.pdf).",
+                taxonomy=taxonomy,
+                main_metric="math_equiv_chain_of_thought",
+                main_split="test",
+            )
+        else:
+            return ScenarioMetadata(
+                name="math_regular",
+                display_name="MATH",
+                description="The MATH benchmark for measuring mathematical problem solving on competition "
+                "math problems [(Hendrycks et al., "
+                "2021)](https://datasets-benchmarks-proceedings.neurips.cc/paper/2021/hash/be83ab3ecd0db773eb2dc1b0a17836a1-Abstract-round2.html).",
+                taxonomy=taxonomy,
+                main_metric="math_equiv",
+                main_split="test",
+            )