crfm-helm 0.5.6__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/scenarios/med_qa_scenario.py

@@ -2,6 +2,7 @@ import json
 import os
 from typing import Dict, List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -12,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     VALID_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -103,3 +105,15 @@ class MedQAScenario(Scenario):
                     instances.append(instance)
 
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="med_qa",
+            display_name="MedQA",
+            description="MedQA is an open domain question answering dataset composed of questions from "
+            "professional medical board exams ([Jin et al. "
+            "2020](https://arxiv.org/pdf/2009.13081.pdf)).",
+            taxonomy=TaxonomyInfo(task="question answering", what="n/a", when="n/a", who="n/a", language="English"),
+            main_metric="quasi_exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/medalign_scenario.py

@@ -1,5 +1,6 @@
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -8,6 +9,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 from helm.benchmark.scenarios.medalign_scenario_helper import return_dataset_dataframe  # type: ignore
 
@@ -92,3 +94,24 @@ class MedalignScenario(Scenario):
     def get_instances(self, output_path: str) -> List[Instance]:
         dataset = return_dataset_dataframe(self.max_length, self.data_path)
         return self.process_tsv(dataset)
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medalign",
+            display_name="MedAlign",
+            short_display_name="MedAlign",
+            description="MedAlign is a benchmark that evaluates a model's ability to interpret and "
+            "follow instructions grounded in longitudinal electronic health records (EHR). "
+            "Each instance includes an event-stream style patient record and a natural "
+            "language question or task, requiring clinically informed reading comprehension "
+            "and reasoning [(Fleming et al., 2023)](https://arxiv.org/abs/2308.14089).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Answer questions and follow instructions over longitudinal EHR",
+                when="Any",
+                who="Clinician, Researcher",
+                language="English",
+            ),
+            main_metric="medalign_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/medalign_scenario_helper.py

@@ -2,22 +2,13 @@
 # type: ignore
 # fmt: off
 
-import ast
-import datetime
 import transformers
-import langchain
-import langchain.prompts
-import lxml.etree
 import os
 import pandas as pd
-import re
 import tiktoken
 
-from langchain_community.retrievers import BM25Retriever
 from tqdm import tqdm
-from typing import Any, Dict, Optional, Union, Callable
-from langchain.schema import Document
-import langchain_community
+from typing import Any, Dict, Optional, Callable
 
 from helm.common.general import check_file_exists
 
@@ -167,102 +158,13 @@ def get_tokenizer(tokenizer_name: str) -> Callable:
     return transformers.AutoTokenizer.from_pretrained(tokenizer_name, legacy=False)
 
 
-def retrieve_most_relevant_visits(ehr_visit_strs, query, target_length, tokenizer):
-    """
-    Retrieve and filter relevant EHR visits based on a query and target length.
-
-    This function retrieves electronic health record (EHR) visit strings, sorts them
-    by relevance using the BM25Retriever, and constructs a list of final documents
-    that fit within a specified character length. The final list ensures that the
-    most important visit isn't cut off and is sorted chronologically.
-
-    Parameters:
-        ehr_visit_strs (list of str): List of EHR visit strings.
-        query (str): Query string to retrieve relevant visits.
-        target_length (int): Maximum total token count for the final list of documents.
-        tokenizer (Callable): Tokenizer that converts text to tokens (used for tracking context length)
-
-    Returns:
-        list[str]: List of EHR visit strings sorted chronologically and constrained by the target length.
-    """
-    ehr_visits=re.split(r'(?=</encounter>\n)',ehr_visit_strs)
-    langchain_docs = [
-        langchain.schema.Document(page_content=doc) for doc in ehr_visits #broken since ehr_visit_strs is one string of all visits
-    ]
-    # `k` is the number of documents to retrieve
-    # We retrieve everything and just use the BM25Retriever to sort the documents
-    retriever = langchain_community.retrievers.BM25Retriever.from_documents(
-        langchain_docs, k=len(langchain_docs)
-    )
-
-    # Invoking the retriever means the most relevant documents are sorted first
-    sorted_docs = retriever.invoke(query)
-
-    # Define the regex pattern to find the start time
-    # pattern = r'start="([\d/]+ [\d:]+)"'
-    pattern = r'start="([\d/]+ [\d:]+ ?[APM]{0,2})"'
-
-    docs = []
-    dts = []
-
-    # Find the startime of the document
-    for doc in sorted_docs:
-        doc_content = doc.page_content
-        start_dt_match = re.search(pattern, doc_content)
-        if start_dt_match:
-            start_dt = start_dt_match.group(1)
-            parsed = False
-            # Try different date formats
-            for fmt in (
-                "%m/%d/%y %I:%M %p",
-                "%m/%d/%Y %I:%M %p",
-                "%m/%d/%y %H:%M",
-                "%m/%d/%Y %H:%M",
-            ):
-                try:
-                    dts.append(datetime.datetime.strptime(start_dt, fmt))
-                    parsed = True
-                    break
-                except ValueError:
-                    continue
-            if not parsed:
-                print(f"Error parsing date: {start_dt}")
-                continue
-        else:
-            print(f"Start time not found., {doc_content}")
-            dts.append(datetime.datetime.min)
-        docs.append(doc_content)
-
-    final_docs = []
-    current_length = 0
-
-    # Add documents until we exceed the allocated context length
-    for i in range(len(docs)):
-        doc_content = docs[i]
-        doc_length = len(tokenizer.encode(doc_content))
-        final_docs.append((dts[i], doc_content))
-        current_length += doc_length
-        if current_length > target_length:
-            break
-
-    # Sort final_docs chronologically
-    final_docs.sort(key=lambda x: x[0])
-
-    # Extract only the document content for the final output
-    final_docs_content = [doc_content for _, doc_content in final_docs]
-
-    return final_docs_content
-
-
-
 def pack_and_trim_prompts(
     instructions: Dict[int, Dict[str, str]],
     ehrs: Dict[int, str],
-    prompt_template: langchain.prompts.PromptTemplate,
+    prompt_string: str,
     context_length: int,
     generation_length: int,
     tokenizer: Any,
-    use_RAG: bool = True,
     verbose: bool = False,
     include_ehr: bool = True,
 ) -> Dict[int, str]:
@@ -276,26 +178,15 @@ def pack_and_trim_prompts(
         patient_id = int(instructions[instruction_id]["patient_id"])
         relevant_ehr = ehrs[patient_id]
 
-        # Calculate how many tokens of EHR we can include in the prompt
         num_tokens_instruction = len(tokenizer.encode(instruction))
-        num_tokens_prompt_template = len(tokenizer.encode(prompt_template.template))
+        num_tokens_prompt_template = len(tokenizer.encode(prompt_string))
         if include_ehr:
             target_ehr_length = context_length - generation_length - num_tokens_prompt_template - num_tokens_instruction
         else:
             target_ehr_length = 0
         if target_ehr_length <= 0:
-            prompt_with_truncated_ehr = prompt_template.format(question=instruction, ehr="")
+            prompt_with_truncated_ehr = prompt_string.format(question=instruction, ehr="")
         else:
-            if use_RAG:
-                # Return a list of the most relevant visit strings
-                most_relevant_visits = retrieve_most_relevant_visits(
-                    ehr_visit_strs=relevant_ehr,
-                    query=instruction,
-                    target_length=target_ehr_length,
-                    tokenizer=tokenizer,
-                )
-                relevant_ehr = "\n".join(most_relevant_visits)
-
             # Do a first pass with a fast tokenizer
             fast_tokenizer = tiktoken.get_encoding("cl100k_base")
             fast_encoded = fast_tokenizer.encode(relevant_ehr)
@@ -307,13 +198,17 @@ def pack_and_trim_prompts(
                 encoded_ehr = tokenizer.encode(fast_truncated_ehr)
                 truncated_encoded_ehr = encoded_ehr[-target_ehr_length:]
                 truncated_ehr = tokenizer.decode(truncated_encoded_ehr)
-                prompt_with_truncated_ehr = prompt_template.format(question=instruction, ehr=truncated_ehr)
+                prompt_with_truncated_ehr = prompt_string.format(question=instruction, ehr=truncated_ehr)
+            else:
+                # If the fast encoding is still too long, just use the full EHR up to allowed length
+                truncated_ehr = fast_tokenizer.decode(fast_encoded[-target_ehr_length:])
+                prompt_with_truncated_ehr = prompt_string.format(question=instruction, ehr=truncated_ehr)
 
-                prompts_map[instruction_id] = prompt_with_truncated_ehr
+        prompts_map[instruction_id] = prompt_with_truncated_ehr
 
-                if verbose:
-                    print(prompt_with_truncated_ehr)
-                    print("~" * 20)
+        if verbose:
+            print(prompt_with_truncated_ehr)
+            print("~" * 20)
     return prompts_map
 
 
@@ -322,7 +217,6 @@ def preprocess_prompts(
     generation_length,
     path_to_instructions,
     path_to_ehrs,
-    use_RAG,
     include_ehr,
     tokenizer,
     codes_only=False,
@@ -347,16 +241,18 @@ def preprocess_prompts(
 
     # CONSTRUCT & TRUNCATE PROMPTS #
     print("Constructing prompts using instructions and EHRs...")
-    prompt_string="Instruction: Answer the following question based on the EHR:\n\nEHR: {ehr}\n\nQuestion: {question}\n\nAnswer:"
-    prompt_template = langchain.prompts.PromptTemplate.from_template(prompt_string)
+    prompt_string = (
+        "Instruction: Answer the following question based on the EHR:\n\n"
+        "EHR: {ehr}\n\nQuestion: {question}\n\nAnswer:"
+    )
+
     filled_prompts = pack_and_trim_prompts(
         instructions=instructions,
         ehrs=ehrs,
-        prompt_template=prompt_template,
+        prompt_string=prompt_string,
         context_length=target_context_length,
         generation_length=generation_length,
         tokenizer=tokenizer,
-        use_RAG=use_RAG,
         verbose=False,
         include_ehr=include_ehr,
     )
@@ -415,7 +311,6 @@ def return_dataset_dataframe(max_length: int, data_path: str) -> pd.DataFrame:
     path_to_ehrs = os.path.join(data_path, "medalign_ehr_xml")
     path_to_reference_responses = os.path.join(data_path, "clinician-instruction-responses.tsv")
     check_file_exists(path_to_reference_responses, msg=f"[MedAlignScenario] Required clinician responses file not found: '{path_to_reference_responses}'")
-    use_RAG = False
     include_ehr = True
     tokenizer = "tiktoken"
 
@@ -424,7 +319,6 @@ def return_dataset_dataframe(max_length: int, data_path: str) -> pd.DataFrame:
         generation_length=generation_length,
         path_to_instructions=path_to_instructions,
         path_to_ehrs=path_to_ehrs,
-        use_RAG=use_RAG,
         include_ehr=include_ehr,
         tokenizer=tokenizer,
     )

helm/benchmark/scenarios/medbullets_scenario.py

@@ -3,6 +3,7 @@ import csv
 import sys
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     TEST_SPLIT,
@@ -11,6 +12,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
     Reference,
     Scenario,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_file_downloaded
 
@@ -143,3 +145,23 @@ class MedBulletsScenario(Scenario):
             csv_path = self.download_csv(output_path, split_suffix)
             instances.extend(self.process_csv(csv_path, split))
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medbullets",
+            display_name="Medbullets",
+            description="Medbullets is a benchmark of USMLE-style medical questions designed to assess "
+            "a model's ability to understand and apply clinical knowledge. Each question is "
+            "accompanied by a patient scenario and five multiple-choice options, similar to "
+            "those found on Step 2 and Step 3 board exams [(MedBullets, "
+            "2025)](https://step2.medbullets.com).",
+            taxonomy=TaxonomyInfo(
+                task="Question answering",
+                what="Medical knowledge testing",
+                when="Any",
+                who="Medical student, . Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/medcalc_bench_scenario.py

@@ -1,6 +1,7 @@
 from typing import Dict, List
 from datasets import load_dataset
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -10,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     PassageQuestionInput,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -125,3 +127,23 @@ class MedCalcBenchScenario(Scenario):
             instances.extend(self.process_csv(data, split))
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medcalc_bench",
+            display_name="MedCalc-Bench",
+            description="MedCalc-Bench is a benchmark designed to evaluate models on their ability to "
+            "compute clinically relevant values from patient notes. Each instance consists "
+            "of a clinical note describing the patient's condition, a diagnostic question "
+            "targeting a specific medical value, and a ground truth response. [(Khandekar "
+            "et al., 2024)](https://arxiv.org/abs/2406.12036).",
+            taxonomy=TaxonomyInfo(
+                task="Computational reasoning",
+                what="Compute a specific medical value from a patient note",
+                when="Any",
+                who="Clinician, Researcher",
+                language="English",
+            ),
+            main_metric="medcalc_bench_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/medec_scenario.py

@@ -1,6 +1,7 @@
 import csv
 import os
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_file_downloaded
 
@@ -123,3 +125,24 @@ class MedecScenario(Scenario):
         instances.extend(self.process_csv(test_csv, TEST_SPLIT))
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medec",
+            display_name="Medec",
+            description="Medec is a benchmark composed of clinical narratives that include either "
+            "correct documentation or medical errors. Each entry includes sentence-level "
+            "identifiers and an associated correction task. The model must review the "
+            "narrative and either identify the erroneous sentence and correct it, or "
+            "confirm that the text is entirely accurate [(Abacha et al., "
+            "2025)](https://arxiv.org/abs/2412.19260).",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Detect and correct errors in medical narratives",
+                when="Any",
+                who="Researcher, Clinician",
+                language="English",
+            ),
+            main_metric="medec_error_flag_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/medhallu_scenario.py

@@ -1,6 +1,7 @@
 from typing import List
 from datasets import load_dataset
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Output,
     Input,
+    ScenarioMetadata,
 )
 
 
@@ -70,3 +72,24 @@ Answer: {answer}
             )
             instances.append(hallucinated_instance)
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medhallu",
+            display_name="MedHallu",
+            description="MedHallu is a benchmark focused on evaluating factual correctness in "
+            "biomedical question answering. Each instance contains a PubMed-derived "
+            "knowledge snippet, a biomedical question, and a model-generated answer. The "
+            "task is to classify whether the answer is factually correct or contains "
+            "hallucinated (non-grounded) information. This benchmark is designed to assess "
+            "the factual reliability of medical language models.",
+            taxonomy=TaxonomyInfo(
+                task="Classification",
+                what="Verify whether answers to questions from PubMed articles are " "factual or hallucinated",
+                when="Any",
+                who="Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/medhelm/judges.yaml

@@ -0,0 +1,14 @@
+# The judges to be used for evaluating the note summary scenario.
+# name: The short name for the judge.
+# model: The field value matching the 'model_name' field under model_deployments.yaml
+# model_deployment: The field value matching the 'name' under model_deployments.yaml.
+judges:
+  - name: "gpt"
+    model: "openai/gpt-4o-2024-05-13"
+    model_deployment: "stanfordhealthcare/gpt-4o-2024-05-13"
+  - name: "llama"
+    model: "meta/llama-3.3-70b-instruct"
+    model_deployment: "stanfordhealthcare/llama-3.3-70b-instruct"
+  - name: "claude"
+    model: "anthropic/claude-3-7-sonnet-20250219"
+    model_deployment: "stanfordhealthcare/claude-3-7-sonnet-20250219"

helm/benchmark/scenarios/medhelm_configurable_scenario.py

@@ -0,0 +1,101 @@
+import string
+import json
+import pandas as pd
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.run_specs.medhelm.benchmark_config import get_benchmark_config_from_path
+from helm.common.general import check_file_exists
+
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    CORRECT_TAG,
+    Reference,
+    Input,
+    Output,
+    TEST_SPLIT,
+    ScenarioMetadata,
+)
+
+
+class MedHELMConfigurableScenario(Scenario):
+    """
+    MedHELM configuratble scenario
+    """
+
+    tags = ["biomedical"]
+
+    def __init__(self, name: str, config_path: str):
+        super().__init__()
+        self.benchmark_config = get_benchmark_config_from_path(config_path)
+        self.name = name
+        self.description = self.benchmark_config.description
+
+    def get_columns_in_template(self, template: str) -> List[str]:
+        """
+        Extract field names from a template string using Python's Formatter.
+        Example: "Name: {name}, Age: {age}" → ["name", "age"]
+        """
+        formatter = string.Formatter()
+        return [fname for _, fname, _, _ in formatter.parse(template) if fname]
+
+    def populate_template(self, template: str, row: pd.Series, fields: List[str]) -> str:
+        """
+        Populate the template with values from the row using format_map.
+        Missing fields default to empty string.
+        """
+        mapping = {field: row.get(field, "") for field in fields}
+        return template.format_map(mapping)
+
+    def get_references(self, row: pd.Series) -> List[Reference]:
+        references: List[Reference] = []
+        if "correct_answer" in row:
+            references.append(Reference(Output(text=row["correct_answer"]), tags=[CORRECT_TAG]))
+        if "incorrect_answers" in row:
+            for incorrect_answer in row["incorrect_answers"]:
+                references.append(Reference(Output(text=incorrect_answer), tags=[]))
+        return references
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        check_file_exists(self.benchmark_config.prompt_file, msg=f"Prompt file for {self.name} does not exist")
+        check_file_exists(self.benchmark_config.dataset_file, msg=f"Dataset file for {self.name} does not exist")
+        instances: List[Instance] = []
+        df = pd.read_csv(self.benchmark_config.dataset_file)
+        if "correct_answer" not in df.columns:
+            if not self._is_llm_as_judge() or len(self.benchmark_config.metrics) > 1:
+                raise ValueError(
+                    "Dataset must contain 'correct_answer' column unless using jury_score as the only metric."
+                )
+        if "incorrect_answers" in df.columns:
+            df["incorrect_answers"] = df["incorrect_answers"].apply(json.loads)
+        with open(self.benchmark_config.prompt_file, "r") as f:
+            template = f.read()
+        fields = self.get_columns_in_template(template)
+        for _, row in df.iterrows():
+            filled = self.populate_template(template, row, fields)
+            prompt = Input(text=filled)
+            instances.append(Instance(input=prompt, references=self.get_references(row), split=TEST_SPLIT))
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name=self.name,
+            display_name=self.name,
+            description=self.description,
+            taxonomy=TaxonomyInfo(
+                task="",
+                what="",
+                when="",
+                who="",
+                language="",
+            ),
+            main_metric=self.benchmark_config.main_metric.name,
+            main_split="test",
+        )
+
+    def _is_llm_as_judge(self) -> bool:
+        for metric in self.benchmark_config.metrics:
+            if metric.name == "jury_score":
+                return True
+        return False

helm/benchmark/scenarios/medi_qa_scenario.py

@@ -1,6 +1,7 @@
 from typing import Dict, List
 from datasets import load_dataset
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.scenarios.scenario import (
     Scenario,
@@ -10,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     CORRECT_TAG,
     Input,
     Output,
+    ScenarioMetadata,
 )
 
 
@@ -109,3 +111,24 @@ class MediQAScenario(Scenario):
             instances.extend(self.process_csv(data, split))
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medi_qa",
+            display_name="MEDIQA",
+            description="MEDIQA is a benchmark designed to evaluate a model's ability to retrieve and "
+            "generate medically accurate answers to patient-generated questions. Each "
+            "instance includes a consumer health question, a set of candidate answers (used "
+            "in ranking tasks), relevance annotations, and optionally, additional context. "
+            "The benchmark focuses on supporting patient understanding and accessibility in "
+            "health communication.",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Retrieve and rank answers based on medical question " "understanding",
+                when="Any",
+                who="Clinician, Medical Student",
+                language="English",
+            ),
+            main_metric="medi_qa_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/medication_qa_scenario.py

@@ -3,9 +3,19 @@ from typing import List
 
 import pandas as pd
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import ensure_file_downloaded
 
-from helm.benchmark.scenarios.scenario import CORRECT_TAG, TEST_SPLIT, Input, Instance, Output, Reference, Scenario
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Instance,
+    Output,
+    Reference,
+    Scenario,
+    ScenarioMetadata,
+)
 
 
 class MedicationQAScenario(Scenario):
@@ -64,3 +74,23 @@ class MedicationQAScenario(Scenario):
         ]
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medication_qa",
+            display_name="MedicationQA",
+            description="MedicationQA is a benchmark composed of open-ended consumer health questions "
+            "specifically focused on medications. Each example consists of a free-form "
+            "question and a corresponding medically grounded answer. The benchmark "
+            "evaluates a model's ability to provide accurate, accessible, and informative "
+            "medication-related responses for a lay audience.",
+            taxonomy=TaxonomyInfo(
+                task="Question answering",
+                what="Answer consumer medication-related questions",
+                when="Any",
+                who="Patient, Pharmacist",
+                language="English",
+            ),
+            main_metric="medication_qa_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/melt_scenarios.py

@@ -439,13 +439,13 @@ class MELTMATHScenario(Scenario):
         for split, split_name in zip([TRAIN_SPLIT, TEST_SPLIT], ["train", "test"]):
             if split == TRAIN_SPLIT and self.use_official_examples:
                 train_instances = [
-                    ("Kết quả của $\left(\\frac{7}{8}\\right)^3 \cdot \left(\\frac{7}{8}\\right)^{-3}$ là gì?", "1"),
+                    ("Kết quả của $\\left(\\frac{7}{8}\\right)^3 \\cdot \\left(\\frac{7}{8}\\right)^{-3}$ là gì?", "1"),
                     (
                         "Có bao nhiêu cách chọn 4 quyển sách từ một kệ sách có 6 quyển,"
                         + " nếu thứ tự các cuốn sách được chọn không quan trọng?",
                         "15",
                     ),
-                    ("Tìm khoảng cách giữa các điểm $(2,1,-4)$ và $(5,8,-3).$", "\sqrt{59}"),
+                    ("Tìm khoảng cách giữa các điểm $(2,1,-4)$ và $(5,8,-3).$", "\\sqrt{59}"),
                     (
                         "Các mặt của khối xúc xắc bát diện được dán nhãn bằng các số từ $1$ đến $8$."
                         + " Xác suất tung một cặp xúc xắc bát diện để được tổng số bằng $15$ là bao nhiêu?"