crfm-helm 0.5.6__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/healthqa_br_scenario.py

@@ -0,0 +1,80 @@
+from typing import Any, List
+import re
+from pathlib import Path
+from datasets import load_dataset
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class HEALTHQA_BR_Scenario(Scenario):
+    """
+    HealthQA-BR is a large-scale benchmark designed to evaluate the clinical knowledge of Large Language Models (LLMs)
+    within the Brazilian Unified Health System (SUS) context. It comprises 5,632 multiple-choice questions sourced from
+    nationwide licensing exams and residency tests, reflecting real challenges faced by Brazil's public health sector.
+    Unlike benchmarks focused on the U.S. medical landscape, HealthQA-BR targets the Brazilian healthcare ecosystem,
+    covering a wide range of medical specialties and interdisciplinary professions such as nursing, dentistry,
+    psychology, social work, pharmacy, and physiotherapy. This comprehensive approach enables a detailed assessment
+    of AI models’ ability to collaborate effectively in the team-based patient care typical of SUS.
+    """
+
+    name = "healthqa_br"
+    description = "MQA benchmark with questions from Brazilian entrance exams"
+    tags = ["knowledge", "multiple_choice", "pt-br"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Download the raw data and read all the dialogues
+        dataset: Any
+        # Read all the instances
+        instances: List[Instance] = []
+        cache_dir = str(Path(output_path) / "data")
+
+        dataset = load_dataset("Larxel/healthqa-br", cache_dir=cache_dir)
+        for example in dataset["train"]:
+            question_choices = example["question"]
+            answer = example["answer"].strip().upper()
+
+            # Separate the question statement from the alternatives
+            question_text, choices_text = self.split_question_and_choices(question_choices)
+
+            # Extract alternatives from text choices_text
+            pattern = r"'([A-Z])':\s*'([^']+)'"
+            matches = re.findall(pattern, choices_text)
+            answers_dict = {label: text for label, text in matches}
+
+            if answer not in answers_dict:
+                continue
+
+            correct_answer_text = answers_dict[answer]
+
+            def answer_to_reference(answer: str) -> Reference:
+                return Reference(Output(text=answer), tags=[CORRECT_TAG] if correct_answer_text == answer else [])
+
+            instance = Instance(
+                input=Input(text=question_text),
+                split=TEST_SPLIT,
+                references=[answer_to_reference(text) for text in answers_dict.values()],
+            )
+            instances.append(instance)
+        return instances
+
+    def split_question_and_choices(self, full_text: str):
+        # Search for the first occurrence of the alternative pattern
+        match = re.search(r"\n'[A-Z]':\s*'.+", full_text)
+        if match:
+            # Everything before the alternatives
+            question_part = full_text[: match.start()].strip()
+            # All of the alternatives (from match to end)
+            choices_part = full_text[match.start() :].strip()
+        else:
+            # If you don't find a pattern, consider everything as a question, and no alternative.
+            question_part = full_text.strip()
+            choices_part = ""
+
+        return question_part, choices_part

helm/benchmark/scenarios/infinite_bench_en_mc_scenario.py

@@ -0,0 +1,90 @@
+import os
+import re
+from typing import List
+
+from datasets import load_dataset, Features, Value, Sequence, Dataset
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Input,
+    Reference,
+    Output,
+    CORRECT_TAG,
+    TEST_SPLIT,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class InfiniteBenchEnMCScenario(Scenario):
+    """InfiniteBench En.MC
+
+    InfiniteBench is a benchmark tailored for evaluating the capabilities of language models to process,
+    understand, and reason over long contexts (100k+ tokens). InfiniteBench En.MC is a subset of
+    InfiniteBench that requires models to perform multiple-choice question answering on questions that necessitate
+    long-range dependency and reasoning, beyond simple short passage retrieval.
+    """
+
+    name = "infinite_bench_en_mc"
+    description = "∞Bench En.MC is a multiple-choice question answering task that necessitates long-range dependency and reasoning. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))"  # noqa: E501
+    tags = ["question_answering"]
+
+    def __init__(self, max_num_words: int):
+        self.max_num_words = max_num_words
+        super().__init__()
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Get InfiniteBench from HuggingFace
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+
+        # Define the features schema
+        ft = Features(
+            {
+                "id": Value("int64"),
+                "context": Value("string"),
+                "input": Value("string"),
+                "answer": Sequence(Value("string")),
+                "options": Sequence(Value("string")),
+            }
+        )
+
+        # Load the dataset with the specified features
+        dataset = load_dataset(
+            "xinrongzhang2022/InfiniteBench",
+            split="longbook_choice_eng",
+            features=ft,
+            revision="90f0394333616266d9fe85824ceaf505093cbaa5",
+        )
+
+        assert isinstance(dataset, Dataset)
+
+        def count_words(text: str) -> int:
+            return len(re.split(r"\s+", text.strip()))
+
+        dataset = dataset.filter(
+            lambda example: count_words(example["context"])
+            + count_words(example["input"])
+            + sum(count_words(option) for option in example["options"])
+            <= self.max_num_words
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row in dataset:
+            assert len(row["answer"]) == 1
+            id = row["id"]
+            input = Input(text=row["context"] + "\n\n" + row["input"])
+            references = [
+                Reference(Output(text=option), tags=[CORRECT_TAG] if option == row["answer"][0] else [])
+                for option in row["options"]
+            ]
+            instance = Instance(
+                id=id,
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/infinite_bench_en_qa_scenario.py

@@ -26,7 +26,7 @@ class InfiniteBenchEnQAScenario(Scenario):
     """
 
     name = "infinite_bench_en_qa"
-    description = "∞Bench En.QA is a summarization task that requires generating a concise summary of a novel. ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))"  # noqa: E501
+    description = "∞Bench En.QA is an open-ended question answering task that necessitates long-range dependency and reasoning.  ([Zhang et al., 2024](https://arxiv.org/abs/2402.13718))"  # noqa: E501
     tags = ["question_answering"]
 
     def __init__(self, max_num_words: int):

helm/benchmark/scenarios/lmkt_scenarios.py

@@ -0,0 +1,288 @@
+"""Cultural alignment evaluation scenario based on Vietnam World Values Survey responses."""
+
+import os
+import json
+import random
+from typing import List
+from datasets import load_dataset
+from huggingface_hub import snapshot_download
+
+from helm.common.hierarchical_logger import hlog, hwarn
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+SUPPORTED_LANGUAGES = ["en", "vi"]
+
+
+class CulturalValueUnderstandingWVSScenario(Scenario):
+    """Cultural values understanding evaluation based on Vietnam World Values Survey responses."""
+
+    name = "cultural_value_understanding_wvs"
+    description = "Evaluates model understanding of cultural values from WVS Wave 7"
+    tags = ["cultural_value_understanding"]
+
+    def __init__(
+        self,
+        language: str,
+        personas_filename: str = "vn_personas_with_filtered_responses.json",
+        questions_filename: str = "wvs_questions_dump.en.json",
+        repo_id: str = "stair-lab/cultural-alignment-dataset",
+        num_personas: int = 10,
+        num_question_variants: int = 2,
+        include_few_shot_examples: bool = False,
+    ):
+        super().__init__()
+        self.language = language
+        self.personas_filename = personas_filename
+        self.questions_filename = questions_filename
+        self.repo_id = repo_id
+        self.num_personas = 10 if isinstance(num_personas, str) else num_personas
+        self.num_question_variants = min(num_question_variants, 4)
+        self.include_few_shot_examples = include_few_shot_examples
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_few_shot_examples(self) -> str:
+        """Return a few-shot learning examples formatted as a string."""
+        examples = """Here are some examples:
+
+Example 1:
+You are a 35 year old male from Vietnam. You have higher education and consider yourself middle class. You are married.
+
+How important is family in your life?
+
+1. Very important
+2. Rather important
+3. Not very important
+4. Not at all important
+
+Please select the option number that best represents your view.
+1
+
+Example 2:
+You are a 28 year old female from Vietnam. You have middle education and consider yourself working class. You are single.
+
+Would you say that most people can be trusted or that you need to be very careful in dealing with people?
+
+1. Most people can be trusted
+2. Need to be very careful
+
+Please select the option number that best represents your view.
+2
+
+Example 3:
+You are a 50 year old male from Vietnam. You have lower education and consider yourself lower class. You are married.
+
+Do you think that homosexuality is justifiable?
+
+1. Never justifiable
+2. Rarely justifiable
+3. Sometimes justifiable
+4. Always justifiable
+
+Please select the option number that best represents your view.
+1
+
+Now answer the following question:
+"""  # noqa: E501
+        return examples
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Generate test instances from Vietnam personas and WVS questions."""
+        instances: List[Instance] = []
+
+        # Download files from Hugging Face Hub
+        repo_local_path = snapshot_download(
+            repo_id=self.repo_id, repo_type="dataset", revision="fe54b6f5d75cfca5377707cd7199e39f517e3a1f"
+        )
+
+        # Load the downloaded files
+        with open(os.path.join(repo_local_path, self.personas_filename), "r", encoding="utf-8") as f:
+            personas = json.load(f)
+
+        with open(os.path.join(repo_local_path, self.questions_filename), "r", encoding="utf-8") as f:
+            questions = json.load(f)
+
+        # Get few-shot examples
+        few_shot_examples = self.get_few_shot_examples() if self.include_few_shot_examples else ""
+
+        # Sample personas
+        sampled_personas = random.sample(personas, min(self.num_personas, len(personas)))
+
+        # Create instances for each persona and question
+        for persona in sampled_personas:
+            # Get demographic info for persona description
+            persona_desc = (
+                f"You are a {persona.get('age', 'adult')} year old {persona.get('sex', 'person')} from Vietnam. "
+            )
+            persona_desc += f"You have {persona.get('education', 'some')} education and consider yourself {persona.get('social_class', 'middle class')}. "  # noqa: E501
+            persona_desc += f"You are {persona.get('marital_status', 'single')}."
+
+            # Process each question this persona answered
+            for qid, human_response in persona.get("responses", {}).items():
+                # Skip if no human response or if it's 0 (which might be a "Don't know" response)
+                if human_response is None:
+                    continue
+
+                # Convert human_response to int (if possible)
+                try:
+                    human_response_int = int(human_response)
+                except (ValueError, TypeError):
+                    # Skip if human_response can't be converted to int
+                    continue
+
+                # Get question info
+                question_data = questions.get(qid, {})
+                if not question_data:
+                    continue
+
+                # Get options directly from question data
+                q_options = question_data.get("options", [])
+                if not q_options:
+                    continue
+
+                # Skip if human_response is out of range
+                if human_response_int < 0 or human_response_int > len(q_options):
+                    continue
+
+                # Special handling for "Don't know" or zero responses
+                if human_response_int == 0:
+                    # Some questions might encode "Don't know" as 0
+                    # Skip for now, or you could add special handling
+                    continue
+
+                # Use the predefined question variations
+                question_variants = question_data.get("questions", [])
+                if not question_variants:
+                    question_variants = [f"Question {qid}: {question_data.get('description', '')}"]
+
+                # Use the specified number of variants
+                variants_to_use = min(self.num_question_variants, len(question_variants))
+                selected_variants = question_variants[:variants_to_use]
+
+                # Create instances for each selected question variant
+                for q_text in selected_variants:
+                    # Format the prompt with or without few-shot examples
+                    if self.include_few_shot_examples:
+                        prompt = f"{few_shot_examples}{persona_desc}\n\n{q_text}\n\n"
+                    else:
+                        prompt = f"{persona_desc}\n\n{q_text}\n\n"
+
+                    # Add options from question data - with numbers, not letters
+                    for i, opt in enumerate(q_options, 1):
+                        prompt += f"{i}. {opt}\n"
+
+                    prompt += "\nPlease select the option number that best represents your view. Return only the option number. Do not return anything else."  # noqa: E501
+
+                    # Create a reference with just the human response number
+                    # We don't create multiple references, just use the actual human response
+                    reference = Reference(Output(text=str(human_response_int)), tags=[CORRECT_TAG])
+
+                    # Create the instance
+                    instance = Instance(
+                        input=Input(text=prompt),
+                        references=[reference],
+                        split=TEST_SPLIT,
+                    )
+                    instances.append(instance)
+
+        if not instances:
+            hwarn("No valid instances were created. Check the input data and parameters.")
+        else:
+            hlog(f"Created {len(instances)} instances for cultural value understanding scenario.")
+
+        return instances
+
+
+class SocialNormApplicationNormADScenario(Scenario):
+    """Social norms application evaluation based on Vietnam World Values Survey responses."""
+
+    name = "social_norm_application_normad"
+    description = "Evaluates model understanding of social norms from WVS Wave 7"
+    tags = ["social_norm_application"]
+
+    def __init__(
+        self,
+        language: str,
+    ):
+        super().__init__()
+        self.language = language
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset = load_dataset(
+            "akhilayerukola/NormAd", split="train", revision="311049f5610e932792672e4e13e7df62b6290336"
+        )
+
+        instances: List[Instance] = []
+
+        for item in dataset:
+            input_text = (
+                f"Assumming that below norms and story are from {item['Country'].capitalize()}:\n"
+                f"{item['Background']}\n\n{item['Story']}\n"
+                "Return one of the following options: 'yes', 'no', or 'neutral'. Do not return anything else.\n\n"
+            )
+
+            instance = Instance(
+                input=Input(text=input_text),
+                references=[Reference(Output(text=item["Gold Label"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances
+
+
+class SocialNormExplanationNormADScenario(Scenario):
+    """Social norms explanation evaluation based on Vietnam World Values Survey responses."""
+
+    name = "social_norm_explanation_normad"
+    description = "Evaluates model understanding of social norms from WVS Wave 7"
+    tags = ["social_norm_explanation"]
+
+    def __init__(
+        self,
+        language: str,
+    ):
+        super().__init__()
+        self.language = language
+
+        if self.language not in SUPPORTED_LANGUAGES:
+            raise ValueError(f"Unsupported language: {self.language}. Supported languages are: {SUPPORTED_LANGUAGES}")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        dataset = load_dataset(
+            "akhilayerukola/NormAd", split="train", revision="311049f5610e932792672e4e13e7df62b6290336"
+        )
+
+        instances: List[Instance] = []
+
+        for item in dataset:
+            if not item["Explanation"]:
+                continue
+
+            input_text = (
+                f"Assumming that below norms and story are from {item['Country'].capitalize()}:\n"
+                f"{item['Background']}\n\n{item['Story']}\n"
+                f"The answer is {item['Gold Label']}. "
+                "Briefly explain the reasoning behind this answer in one or two sentences.\n\n"
+            )
+
+            instance = Instance(
+                input=Input(text=input_text),
+                references=[Reference(Output(text=item["Explanation"]), tags=[CORRECT_TAG])],
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/math_scenario.py

@@ -18,13 +18,14 @@ from helm.benchmark.scenarios.scenario import (
 
 
 def remove_boxed(string: str) -> Optional[str]:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
-    Extract the text within a \\boxed{...} environment.
+    Extract the text within a \boxed{...} environment.
 
     Example:
-    >>> remove_boxed(\\boxed{\\frac{2}{3}})
-    \\frac{2}{3}
+        >>> from helm.benchmark.scenarios.math_scenario import *  # NOQA
+        >>> remove_boxed(r'\boxed{\frac{2}{3}}')
+        '\\frac{2}{3}'
     """
     left = "\\boxed{"
     try:
@@ -68,17 +69,17 @@ def last_boxed_only_string(string: str) -> Optional[str]:
 
 
 def _fix_fracs(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat fractions.
 
     Examples:
-    >>> _fix_fracs("\\frac1b")
-    \frac{1}{b}
-    >>> _fix_fracs("\\frac12")
-    \frac{1}{2}
-    >>> _fix_fracs("\\frac1{72}")
-    \frac{1}{72}
+        >>> _fix_fracs(r"\frac1b")
+        '\\frac{1}{b}'
+        >>> _fix_fracs(r"\frac12")
+        '\\frac{1}{2}'
+        >>> _fix_fracs(r"\frac1{72}")
+        '\\frac{1}{72}'
     """
     substrs = string.split("\\frac")
     new_str = substrs[0]
@@ -112,13 +113,13 @@ def _fix_fracs(string: str) -> str:
 
 
 def _fix_a_slash_b(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat fractions formatted as a/b to \\frac{a}{b}.
 
     Example:
-    >>> _fix_a_slash_b("2/3")
-    \frac{2}{3}
+        >>> _fix_a_slash_b(r"2/3")
+        '\\frac{2}{3}'
     """
     if len(string.split("/")) != 2:
         return string
@@ -149,13 +150,13 @@ def _remove_right_units(string: str) -> str:
 
 
 def _fix_sqrt(string: str) -> str:
-    """Source: https://github.com/hendrycks/math
+    r"""Source: https://github.com/hendrycks/math
 
     Reformat square roots.
 
     Example:
-    >>> _fix_sqrt("\\sqrt3")
-    \sqrt{3}
+        >>> _fix_sqrt("\\sqrt3")
+        '\\sqrt{3}'
     """
     if "\\sqrt" not in string:
         return string
@@ -210,7 +211,7 @@ def _strip_string(string: str) -> str:
 
     # remove percentage
     string = string.replace("\\%", "")
-    string = string.replace("\%", "")
+    string = string.replace(r"\%", "")
 
     # " 0." equivalent to " ." and "{0." equivalent to "{." Alternatively, add "0" if "." is the start of the string
     string = string.replace(" .", " 0.")
@@ -391,13 +392,13 @@ class MATHScenario(Scenario):
         for split, split_name in zip([TRAIN_SPLIT, TEST_SPLIT], ["train", "test"]):
             if split == TRAIN_SPLIT and self.use_official_examples:
                 train_instances = [
-                    ("What is $\left(\\frac{7}{8}\\right)^3 \cdot \left(\\frac{7}{8}\\right)^{-3}$?", "1"),
+                    ("What is $\\left(\\frac{7}{8}\\right)^3 \\cdot \\left(\\frac{7}{8}\\right)^{-3}$?", "1"),
                     (
                         "In how many ways can 4 books be selected from a shelf of 6 books"
                         + " if the order in which the books are selected does not matter?",
                         "15",
                     ),
-                    ("Find the distance between the points $(2,1,-4)$ and $(5,8,-3).$", "\sqrt{59}"),
+                    ("Find the distance between the points $(2,1,-4)$ and $(5,8,-3).$", "\\sqrt{59}"),
                     (
                         "The faces of an octahedral die are labeled with digits $1$ through $8$."
                         + " What is the probability, expressed as a common fraction,"