crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/scenarios/medalign_scenario_helper.py

@@ -0,0 +1,326 @@
+# flake8: noqa
+# type: ignore
+# fmt: off
+
+import transformers
+import os
+import pandas as pd
+import tiktoken
+
+from tqdm import tqdm
+from typing import Any, Dict, Optional, Callable
+
+from helm.common.general import check_file_exists
+
+
+def get_instructions(path_to_instructions: str) -> Dict[int, Dict[str, Any]]:
+    """
+    Builds map from Instruction ID to instruction details
+
+    The needed information for creating the map is accomplished by reading
+    a CSV file from the user-specified path.
+
+    The CSV file is expected to contain at least the following columns:
+    - instruction_id: The ID of the instruction.
+    - question: The text of the instruction.
+    - person_id: The ID of the associated patient.
+    - is_selected_ehr: A flag indicating whether the instruction is selected.
+
+    See https://stanfordmedicine.box.com/s/0om9qav2sklb9vaitn0ibye65vgbfx0e
+
+    Parameters:
+        path_to_instructions (str): Path to CSV file containing instructions.
+
+    Returns:
+        Dict[int, Dict[str, Any]]: A dictionary mapping instruction IDs to a
+            dictionary containing instruction text and associated patient ID.
+
+    Raises:
+        FileNotFoundError: If the specified file does not exist.
+        ValueError: If the CSV file does not contain the expected columns.
+    """
+    if not os.path.exists(path_to_instructions):
+        raise FileNotFoundError(
+            f"The specified file {path_to_instructions} does not exist."
+        )
+
+    instructions_df = pd.read_csv(path_to_instructions, sep='\t')
+    required_columns = {
+        "instruction_id",
+        "question",
+        "person_id",
+    }
+    if not required_columns.issubset(instructions_df.columns):
+        raise ValueError(
+            f"The CSV file is missing one or more of the required columns: {required_columns}"
+        )
+
+    selected_instructions_df = instructions_df #.query("is_selected_ehr == 'yes'")
+    instructions_map = {
+        row["instruction_id"]: {
+            "instruction": row["question"],
+            "patient_id": row["person_id"],
+        }
+        for _, row in selected_instructions_df.iterrows()
+    }
+    return instructions_map
+
+
+def extract_patient_id_from_fname(fname: str) -> Optional[int]:
+    """
+    Extracts and returns the patient ID from a given filename.
+
+    The function expects filenames in the format 'EHR_<patient_id>.xml',
+    where <patient_id> is a sequence of digits.
+
+    Parameters:
+        fname (str): The filename from which to extract the patient ID.
+
+    Returns:
+        Optional[int]: The extracted patient ID as an integer, or None if
+                    the filename doesn't match the expected format.
+    """
+    name=fname.split('.')[0]
+    return int(name)
+
+
+def get_ehrs(path_to_ehrs: str) -> Dict[int, str]:
+    """
+    Builds a map from Instruction ID to EHR (Electronic Health Record) timeline.
+
+    EHR timelines are in string format and EHR files are read in from the
+    user-specified directory. Each file in the directory should be named
+    'EHR_<patient_id>.xml', where <patient_id> is a sequence of digits.
+
+    See https://stanfordmedicine.box.com/s/r28wfwwude9rpjtu0szhzegmku8qv2pe
+
+    Parameters:
+        path_to_ehrs (str): The path to the directory containing the EHR files.
+
+    Returns:
+        Dict[int, str]: A dictionary mapping patient IDs to EHR timelines.
+
+    Raises:
+        FileNotFoundError: If the specified directory does not exist.
+    """
+    if not os.path.isdir(path_to_ehrs):
+        raise FileNotFoundError(
+            f"The specified directory {path_to_ehrs} does not exist."
+        )
+
+    ehr_map = {}
+    for fname in os.listdir(path_to_ehrs):
+        pt_id = extract_patient_id_from_fname(fname)
+        if pt_id is None:
+            print(
+                f"Warning: File '{fname}' does not match the expected format "
+                "and will be skipped."
+            )
+            continue
+
+        file_path = os.path.join(path_to_ehrs, fname)
+        with open(file_path, encoding="utf-8", mode="r") as f:
+            ehr = f.read()
+
+        ehr_map[pt_id] = ehr
+    return ehr_map
+
+
+def get_tokenizer(tokenizer_name: str) -> Callable:
+    """
+    Returns a tokenizer based on the given tokenizer name.
+
+    Parameters:
+        tokenizer_name (str): The name of the tokenizer. Acceptable values are:
+            - "tiktoken"
+            - "chatgpt"
+            - "gpt-3.5-turbo"
+            - "gpt-4"
+            - "gpt-4-turbo"
+            - "gpt-4o"
+            - "cl100k_base"
+            - Any valid tokenizer name recognized by the transformers library.
+
+    Returns:
+        Callable: The tokenizer instance.
+    """
+    if tokenizer_name.lower() in [
+        "tiktoken",
+        "chatgpt",
+        "gpt-3.5-turbo",
+        "gpt-4",
+        "gpt-4-turbo",
+        "gpt-4o",
+        "cl100k_base",
+    ]:
+        return tiktoken.get_encoding("cl100k_base")
+    print(tokenizer_name)
+    return transformers.AutoTokenizer.from_pretrained(tokenizer_name, legacy=False)
+
+
+def pack_and_trim_prompts(
+    instructions: Dict[int, Dict[str, str]],
+    ehrs: Dict[int, str],
+    prompt_string: str,
+    context_length: int,
+    generation_length: int,
+    tokenizer: Any,
+    verbose: bool = False,
+    include_ehr: bool = True,
+) -> Dict[int, str]:
+    """
+    Returns:
+        A map from Instruction ID to prompt
+    """
+    prompts_map = {}
+    for instruction_id in tqdm(instructions.keys()):
+        instruction = instructions[instruction_id]["instruction"]
+        patient_id = int(instructions[instruction_id]["patient_id"])
+        relevant_ehr = ehrs[patient_id]
+
+        num_tokens_instruction = len(tokenizer.encode(instruction))
+        num_tokens_prompt_template = len(tokenizer.encode(prompt_string))
+        if include_ehr:
+            target_ehr_length = context_length - generation_length - num_tokens_prompt_template - num_tokens_instruction
+        else:
+            target_ehr_length = 0
+        if target_ehr_length <= 0:
+            prompt_with_truncated_ehr = prompt_string.format(question=instruction, ehr="")
+        else:
+            # Do a first pass with a fast tokenizer
+            fast_tokenizer = tiktoken.get_encoding("cl100k_base")
+            fast_encoded = fast_tokenizer.encode(relevant_ehr)
+            if len(fast_encoded) <= target_ehr_length:
+                fast_encoded_truncated = fast_encoded[-(2 * target_ehr_length) :]
+                fast_truncated_ehr = fast_tokenizer.decode(fast_encoded_truncated)
+
+                # Then do a second pass with the actual tokenizer
+                encoded_ehr = tokenizer.encode(fast_truncated_ehr)
+                truncated_encoded_ehr = encoded_ehr[-target_ehr_length:]
+                truncated_ehr = tokenizer.decode(truncated_encoded_ehr)
+                prompt_with_truncated_ehr = prompt_string.format(question=instruction, ehr=truncated_ehr)
+            else:
+                # If the fast encoding is still too long, just use the full EHR up to allowed length
+                truncated_ehr = fast_tokenizer.decode(fast_encoded[-target_ehr_length:])
+                prompt_with_truncated_ehr = prompt_string.format(question=instruction, ehr=truncated_ehr)
+
+        prompts_map[instruction_id] = prompt_with_truncated_ehr
+
+        if verbose:
+            print(prompt_with_truncated_ehr)
+            print("~" * 20)
+    return prompts_map
+
+
+def preprocess_prompts(
+    target_context_length,
+    generation_length,
+    path_to_instructions,
+    path_to_ehrs,
+    include_ehr,
+    tokenizer,
+    codes_only=False,
+    notes_only=False,
+):
+    print(
+        f"\n\twith target context length = {target_context_length} "
+        f"\n\twith target generation length = {generation_length} "
+    )
+
+    # FETCH INSTRUCTIONS
+    print("Fetching instructions...")
+    instructions = get_instructions(path_to_instructions)
+
+    # FETCH RELEVANT EHRs #
+    print("Fetching patient EHR timelines...")
+    ehrs = get_ehrs(path_to_ehrs)
+
+    # LOAD TOKENIZER #
+    print("Loading tokenizer...")
+    tokenizer = get_tokenizer(tokenizer)
+
+    # CONSTRUCT & TRUNCATE PROMPTS #
+    print("Constructing prompts using instructions and EHRs...")
+    prompt_string = (
+        "Instruction: Answer the following question based on the EHR:\n\n"
+        "EHR: {ehr}\n\nQuestion: {question}\n\nAnswer:"
+    )
+
+    filled_prompts = pack_and_trim_prompts(
+        instructions=instructions,
+        ehrs=ehrs,
+        prompt_string=prompt_string,
+        context_length=target_context_length,
+        generation_length=generation_length,
+        tokenizer=tokenizer,
+        verbose=False,
+        include_ehr=include_ehr,
+    )
+    assert filled_prompts, f"No prompts were found for length: {target_context_length}. Try again with a larger length."
+    # SAVE CONSTRUCTED PROMPTS TO DISK
+    df_rows = []
+    for instruction_id in tqdm(filled_prompts.keys()):
+        row = {}
+        row["instruction_id"] = instruction_id
+        patient_id = instructions[instruction_id]["patient_id"]
+        row["patient_id"] = patient_id
+        row["instruction"] = instructions[instruction_id]["instruction"]
+        row["ehr"] = "".join(ehrs[patient_id])
+        row["prompt"] = filled_prompts[instruction_id]
+        row["context_length"] = target_context_length
+        row["generation_length"] = generation_length
+        df_rows.append(row)
+
+    prompts_df = pd.DataFrame(df_rows)
+    instructionid_to_prompt_map = (
+        prompts_df[["instruction_id", "prompt"]].set_index("instruction_id").to_dict().get("prompt")
+    )
+    instructionid_to_prompt_df = (
+        pd.DataFrame.from_dict(instructionid_to_prompt_map, orient="index", columns=["prompt"])
+        .reset_index()
+        .rename(columns={"index": "instruction_id"})
+    )
+
+    print("...Prompt construction complete")
+    return instructionid_to_prompt_df
+
+
+def add_reference_responses(prompts_df, path_to_reference_responses) -> pd.DataFrame:
+    """
+    Processes a single file for evaluation.
+
+    Parameters:
+    file_path (str): Path to the file to be processed.
+    args (argparse.Namespace): Command line arguments passed to the script.
+
+    Returns:
+    pd.DataFrame: DataFrame containing the processed data.
+    """
+    gold_df = pd.read_csv(path_to_reference_responses, sep='\t')
+    gold_df = gold_df.query("annotator_num == 'Annotator_1'")
+    gold_df = gold_df[["instruction_id", "clinician_response"]]
+    merged_df = gold_df.merge(prompts_df, on="instruction_id", how="inner")
+    return merged_df
+
+
+def return_dataset_dataframe(max_length: int, data_path: str) -> pd.DataFrame:
+    target_context_length = max_length
+    generation_length = 256
+    path_to_instructions = os.path.join(data_path, "clinician-reviewed-model-responses.tsv")
+    check_file_exists(path_to_instructions, msg=f"[MedAlignScenario] Required instructions file not found: '{path_to_instructions}'")
+    path_to_ehrs = os.path.join(data_path, "medalign_ehr_xml")
+    path_to_reference_responses = os.path.join(data_path, "clinician-instruction-responses.tsv")
+    check_file_exists(path_to_reference_responses, msg=f"[MedAlignScenario] Required clinician responses file not found: '{path_to_reference_responses}'")
+    include_ehr = True
+    tokenizer = "tiktoken"
+
+    instructionid_to_prompt_df = preprocess_prompts(
+        target_context_length=target_context_length,
+        generation_length=generation_length,
+        path_to_instructions=path_to_instructions,
+        path_to_ehrs=path_to_ehrs,
+        include_ehr=include_ehr,
+        tokenizer=tokenizer,
+    )
+    medalign_dataframe = add_reference_responses(instructionid_to_prompt_df, path_to_reference_responses)
+    return medalign_dataframe

helm/benchmark/scenarios/medbullets_scenario.py

@@ -0,0 +1,167 @@
+import os
+import csv
+import sys
+from typing import List
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    TEST_SPLIT,
+    Input,
+    Instance,
+    Output,
+    Reference,
+    Scenario,
+    ScenarioMetadata,
+)
+from helm.common.general import ensure_file_downloaded
+
+csv.field_size_limit(sys.maxsize)
+
+
+class MedBulletsScenario(Scenario):
+    """
+    From "Benchmarking Large Language Models on Answering and Explaining Challenging Medical Questions"
+    (Chen et al.), MedBullet is a dataset comprising USMLE Step 2&3 style clinical questions. The dataset
+    is designed to evaluate the performance of LLMs in answering and explaining challenging medical questions,
+    emphasizing the need for explainable AI in medical QA.
+
+    Example from the dataset:
+
+    Question:
+    A 42-year-old woman is enrolled in a randomized controlled trial to study cardiac function in the setting of
+    several different drugs. She is started on verapamil and instructed to exercise at 50% of her VO2 max while
+    several cardiac parameters are being measured. During this experiment, which of the following represents
+    the relative conduction speed through the heart from fastest to slowest?
+
+    A) AV node > ventricles > atria > Purkinje fibers
+    B) Purkinje fibers > ventricles > atria > AV node
+    C) Purkinje fibers > atria > ventricles > AV node
+    D) Purkinje fibers > AV node > ventricles > atria
+
+    Answer:
+    The answer is C. Explanation: The conduction velocity of the structures of the heart is in the following order:
+    Purkinje fibers > atria > ventricles > AV node. A calcium channel blocker such as verapamil would only slow
+    conduction in the AV node.
+
+    @Article{MedBullet,
+    author = {Hanjie Chen and Zhouxiang Fang and Yash Singla and Mark Dredze},
+    title = {Benchmarking Large Language Models on Answering and Explaining Challenging Medical Questions},
+    year = {2023},
+    abstract = {LLMs have demonstrated impressive performance in answering medical questions, such as passing scores
+    on medical licensing examinations. However, medical board exam questions or general clinical questions do not
+    capture the complexity of realistic clinical cases. Moreover, the lack of reference explanations means we cannot
+    easily evaluate the reasoning of model decisions, a crucial component of supporting doctors in making complex
+    medical decisions. To address these challenges, we construct two new datasets: JAMA Clinical Challenge and
+    Medbullets. JAMA Clinical Challenge consists of questions based on challenging clinical cases, while Medbullets
+    comprises USMLE Step 2&3 style clinical questions. Both datasets are structured as multiple-choice
+    question-answering tasks, where each question is accompanied by an expert-written explanation. We evaluate four
+    LLMs on the two datasets using various prompts. Experiments demonstrate that our datasets are harder than
+    previous benchmarks. The inconsistency between automatic and human evaluations of model-generated explanations
+    highlights the need to develop new metrics to support future research on explainable medical QA.}}
+
+    Task:
+    Given a clinical question with multiple-choice options, models must identify the correct answer and generate a
+    response that includes the reasoning, as described in the expert-written explanation.
+    """
+
+    DATASET_DOWNLOAD_BASE_URL = (
+        "https://raw.githubusercontent.com/HanjieChen/ChallengeClinicalQA/refs/heads/main/medbullets/"
+    )
+
+    name = "medbullets"
+    description = (
+        "Medbullets is a benchmark of USMLE-style medical questions designed to assess a"
+        "model’s ability to understand and apply clinical knowledge. Each question is accompanied"
+        "by a patient scenario and five multiple-choice options, similar to those found on"
+        "Step 2 and Step 3 on the US medical licensing exam."
+    )
+    tags = ["reasoning", "biomedical"]
+
+    # Define the possible answer choices
+    POSSIBLE_ANSWER_CHOICES: List[str] = ["A", "B", "C", "D", "E"]
+
+    def __init__(self):
+        super().__init__()
+        # self.splits = {"_op4": TRAIN_SPLIT, "_op5": TEST_SPLIT}
+        # limit to zero shot setting for now
+        self.splits = {"_op5": TEST_SPLIT}
+
+    def download_csv(self, output_path: str, split: str):
+        """Download CSV files for the given split."""
+        csv_path = os.path.join(output_path, f"medbullets{split}.csv")
+        ensure_file_downloaded(
+            source_url=f"{self.DATASET_DOWNLOAD_BASE_URL}/medbullets{split}.csv",
+            target_path=csv_path,
+            unpack=False,
+        )
+        return csv_path
+
+    def process_csv(self, csv_path: str, split: str) -> List[Instance]:
+        """Read and process a CSV file to generate instances."""
+        instances: List[Instance] = []
+        with open(csv_path, "r", encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+            for row in reader:
+                # Validate required fields
+                if not row.get("question") or not row.get("answer_idx") or not row.get("opa"):
+                    print(f"Skipping invalid row: {row}")
+                    continue
+
+                # Map answers to indices
+                option_map = {
+                    "A": row.get("opa", "Not applicable"),
+                    "B": row.get("opb", "Not applicable"),
+                    "C": row.get("opc", "Not applicable"),
+                    "D": row.get("opd", "Not applicable"),
+                    "E": row.get("ope", "Not applicable"),
+                }
+
+                # Correct answer text
+                correct_option = row["answer_idx"]
+
+                # Build references using POSSIBLE_ANSWER_CHOICES
+                references = [
+                    Reference(
+                        Output(text=option_map.get(option, "Not applicable")),
+                        tags=[CORRECT_TAG] if option == correct_option else [],
+                    )
+                    for option in self.POSSIBLE_ANSWER_CHOICES
+                ]
+
+                # Create instance
+                instance = Instance(
+                    input=Input(text=row["question"]),
+                    references=references,
+                    split=split,
+                )
+                instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        """Download and process dataset to generate instances."""
+        instances: List[Instance] = []
+        for split_suffix, split in self.splits.items():
+            csv_path = self.download_csv(output_path, split_suffix)
+            instances.extend(self.process_csv(csv_path, split))
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medbullets",
+            display_name="Medbullets",
+            description="Medbullets is a benchmark of USMLE-style medical questions designed to assess "
+            "a model's ability to understand and apply clinical knowledge. Each question is "
+            "accompanied by a patient scenario and five multiple-choice options, similar to "
+            "those found on Step 2 and Step 3 board exams [(MedBullets, "
+            "2025)](https://step2.medbullets.com).",
+            taxonomy=TaxonomyInfo(
+                task="Question answering",
+                what="Medical knowledge testing",
+                when="Any",
+                who="Medical student, . Researcher",
+                language="English",
+            ),
+            main_metric="exact_match",
+            main_split="test",
+        )

helm/benchmark/scenarios/medcalc_bench_scenario.py

@@ -0,0 +1,149 @@
+from typing import Dict, List
+from datasets import load_dataset
+
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.common.hierarchical_logger import hlog
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    PassageQuestionInput,
+    Output,
+    ScenarioMetadata,
+)
+
+
+class MedCalcBenchScenario(Scenario):
+    """
+    MedCalc-Bench is the first medical calculation dataset used to benchmark
+    LLMs ability to serve as clinical calculators.
+    Each instance in the dataset consists of a patient note, a question asking
+    to compute a specific clinical value, a final answer value, and a step-by-step
+    solution explaining how the final answer was obtained. Our dataset covers 55
+    different calculation tasks. We hope this dataset serves as a call to improve
+    the verbal and computational reasoning skills of LLMs in medical settings.
+
+    This dataset contains a training dataset of 10,053 instances and a testing
+    dataset of 1,047 instances.
+
+    Dataset: https://huggingface.co/datasets/ncbi/MedCalc-Bench-v1.0
+    Paper: https://arxiv.org/abs/2406.12036
+
+    Sample Prompt:
+        Given a patient note and a clinical question, compute the requested medical value.
+        Be as concise as possible.
+
+        Patient note: A 70-year-old female was rushed into the ICU due to respiratory distress,
+        following which she was promptly put on mechanical ventilation. Her delivered oxygen fell
+        to 51 % FiO₂; meanwhile, her partial pressure of oxygen (PaO₂) registered at 74 mm Hg.
+        She was conscious but visibly disoriented with a functional Glasgow Coma Score of 12.
+        She was hypotensive with blood pressure of 91/70 mm Hg. Multiple vasopressors are being administered
+        simultaneously including DOPamine at 4 mcg/kg/min, norEPINEPHrine at 0.06 mcg/kg/min,
+        DOBUTamine at 3 mcg/kg/min, and EPINEPHrine at 0.03 mcg/kg/min. Laboratory evaluations
+        revealed mild renal impairment with creatinine levels slightly elevated at 1.6 mg/dL
+        and a bilirubin level of 1.9 mg/dL. Her platelet count was found to be 165,000/µL.
+        Her daily urine output of 950 mL.
+        Question: What is the patient's Sequential Organ Failure Assessment (SOFA) Score?
+
+        Answer:
+
+    @misc{khandekar2024medcalcbench,
+        title={MedCalc-Bench: Evaluating Large Language Models for Medical Calculations},
+        author={
+            Nikhil Khandekar and Qiao Jin and Guangzhi Xiong and Soren Dunn and Serina S Applebaum and
+            Zain Anwar and Maame Sarfo-Gyamfi and Conrad W Safranek and Abid A Anwar and Andrew Zhang and
+            Aidan Gilson and Maxwell B Singer and Amisha Dave and Andrew Taylor and Aidong Zhang and
+            Qingyu Chen and Zhiyong Lu
+        },
+        year={2024},
+        eprint={2406.12036},
+        archivePrefix={arXiv},
+        primaryClass={
+            id='cs.CL' full_name='Computation and Language' is_active=True alt_name='cmp-lg'
+            in_archive='cs' is_general=False description='Covers natural language processing.
+            Roughly includes material in ACM Subject Class I.2.7. Note that work on artificial
+            languages (programming languages, logics, formal systems) that does not explicitly
+            address natural-language issues broadly construed (natural-language processing, computational
+            linguistics, speech, text retrieval, etc.) is not appropriate for this area.'
+        }
+    }
+    """
+
+    name = "medcalc_bench"
+    description = (
+        "MedCalc-Bench is a benchmark designed to evaluate models on their ability to compute"
+        "clinically relevant values from patient notes. Each instance consists of a clinical note"
+        "describing the patient's condition, a diagnostic question targeting a specific medical"
+        "value, and a ground truth response."
+    )
+    tags = ["knowledge", "reasoning", "biomedical"]
+
+    def __init__(self):
+        super().__init__()
+
+    def process_csv(self, data, split: str) -> List[Instance]:
+        instances: List[Instance] = []
+        hlog(f"Processing data for {split} split")
+        for row in data:
+            question = row["Question"]
+            ground_truth_answer = row["Ground Truth Answer"]
+            patient_note = row["Patient Note"]
+            id = row["Row Number"]
+
+            prompt = PassageQuestionInput(
+                passage=patient_note + "\n", question=question + "\n", passage_prefix="Patient note: "
+            )
+
+            extra_data = {
+                "category": row["Category"],
+                "upper_limit": row["Upper Limit"],
+                "lower_limit": row["Lower Limit"],
+            }
+
+            instance = Instance(
+                input=prompt,
+                references=[Reference(Output(text=ground_truth_answer), tags=[CORRECT_TAG])],
+                extra_data=extra_data,
+                split=split,
+                id=id,
+            )
+            instances.append(instance)
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        # Load the MedCalc-Bench dataset from Hugging Face
+        dataset = load_dataset("ncbi/MedCalc-Bench-v1.0")
+
+        # Process all the instances - limit to zero shot setting
+        instances: List[Instance] = []
+        splits: Dict[str, str] = {
+            # "train": TRAIN_SPLIT,
+            "test": TEST_SPLIT,
+        }
+        for hf_split, split in splits.items():
+            data = dataset[hf_split]
+            instances.extend(self.process_csv(data, split))
+
+        return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="medcalc_bench",
+            display_name="MedCalc-Bench",
+            description="MedCalc-Bench is a benchmark designed to evaluate models on their ability to "
+            "compute clinically relevant values from patient notes. Each instance consists "
+            "of a clinical note describing the patient's condition, a diagnostic question "
+            "targeting a specific medical value, and a ground truth response. [(Khandekar "
+            "et al., 2024)](https://arxiv.org/abs/2406.12036).",
+            taxonomy=TaxonomyInfo(
+                task="Computational reasoning",
+                what="Compute a specific medical value from a patient note",
+                when="Any",
+                who="Clinician, Researcher",
+                language="English",
+            ),
+            main_metric="medcalc_bench_accuracy",
+            main_split="test",
+        )