crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/scenarios/melt_translation_scenario.py

@@ -0,0 +1,152 @@
+from typing import Any, Dict, List, Optional
+
+from datasets import load_dataset, Dataset
+from helm.common.hierarchical_logger import htrack_block
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MELTTranslationScenario(Scenario):
+    name = "melt_translation"
+    description = "Machine Translation scenario."
+    tags = ["machine_translation"]
+
+    def __init__(
+        self,
+        dataset_name: str,
+        revision: str,
+        source_language: str,
+        target_language: str,
+        subset: Optional[str] = None,
+        splits: Optional[Dict[str, str]] = None,
+    ):
+        """Initializes the question answering scenario.
+
+        Args:
+            dataset_name: The name of the dataset.
+            revision: The revision of the dataset to use.
+            source_language: The source language to use.
+            target_language: The target language to use.
+            subset: The subset of the dataset to use. Defaults to "".
+            splits: The splits to use for the dataset. Defaults to None.
+        """
+        super().__init__()
+        self.MAX_TRAIN_INSTANCES = 20_000
+        valid_languages = set(["vi", "en"])
+        self.dataset_name = dataset_name
+        self.subset = subset
+        self.revision = revision
+        self.splits = splits
+        self.source_language = source_language
+        self.target_language = target_language
+        if self.source_language not in valid_languages or self.target_language not in valid_languages:
+            raise ValueError("Supported languages: vi, en.")
+        if self.source_language == self.target_language:
+            raise ValueError("The source language and the target language should be different.")
+        if self.source_language != "en" and self.target_language != "en":
+            raise ValueError("One of the languages should be English.")
+
+    def get_instances_for_splits(self, splits: Dict[str, str]) -> List[Instance]:
+        """
+        Helper for generating instances for a split.
+        Args:
+            splits (dict): Which splits to partition the data into.
+        Returns:
+            List[Instance]: Instances from the file for the specified split.
+        """
+        with htrack_block("Loading the HuggingFace dataset. The first time could take several minutes."):
+            hf_dataset: Any = load_dataset(
+                self.dataset_name,
+                self.subset,
+                revision=self.revision,
+                trust_remote_code=True,
+            )
+
+        instances: List[Instance] = []
+
+        for dataset_split_name, helm_split_name in splits.items():
+            if helm_split_name == TRAIN_SPLIT:
+                hf_dataset[dataset_split_name] = hf_dataset[dataset_split_name].shuffle(seed=42)[
+                    : self.MAX_TRAIN_INSTANCES
+                ]
+                hf_dataset[dataset_split_name] = Dataset.from_dict(hf_dataset[dataset_split_name])
+
+            for example in hf_dataset[dataset_split_name]:
+                source_sentence = example[self.source_language]
+                target_sentence = example[self.target_language]
+                instances.append(
+                    Instance(
+                        input=Input(text=source_sentence),
+                        references=[Reference(Output(text=target_sentence), tags=[CORRECT_TAG])],
+                        split=helm_split_name,
+                    )
+                )
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        if self.splits is None:
+            splits = {"train": TRAIN_SPLIT, "validation": VALID_SPLIT, "test": TEST_SPLIT}
+        else:
+            splits = {}
+            if "train" in self.splits:
+                splits[self.splits[TRAIN_SPLIT]] = TRAIN_SPLIT
+            if "validation" in self.splits:
+                splits[self.splits[VALID_SPLIT]] = VALID_SPLIT
+            if "test" in self.splits:
+                splits[self.splits[TEST_SPLIT]] = TEST_SPLIT
+
+        instances: List[Instance] = self.get_instances_for_splits(splits=splits)
+        return instances
+
+
+class MELTTranslationOPUS100Scenario(MELTTranslationScenario):
+    """
+    Scenario for the OPUS100 dataset.
+    """
+
+    name = "melt_translation_opus100"
+    description = "OPUS100 dataset for machine translation."
+    tags = ["machine_translation"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="vietgpt/opus100_envi",
+            revision="45df06fb0b31edc882d7c8d34389261f995e5208",
+            splits={
+                TRAIN_SPLIT: "train",
+                VALID_SPLIT: "validation",
+                TEST_SPLIT: "test",
+            },
+            **kwargs,
+        )
+
+
+class MELTTranslationPhoMTScenario(MELTTranslationScenario):
+    """
+    Scenario for the PhoMT dataset.
+    """
+
+    name = "melt_translation_phomt"
+    description = "PhoMT dataset for machine translation."
+    tags = ["machine_translation"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="ura-hcmut/PhoMT",
+            revision="74386685db01dc038860ff0a90d9f5fbde284bf7",
+            splits={
+                TRAIN_SPLIT: "train",
+                VALID_SPLIT: "validation",
+                TEST_SPLIT: "test",
+            },
+            **kwargs,
+        )

helm/benchmark/scenarios/mental_health_scenario.py

@@ -9,6 +9,7 @@ from helm.benchmark.scenarios.scenario import (
     PassageQuestionInput,
     Output,
 )
+from helm.common.general import check_file_exists
 
 
 class MentalHealthScenario(Scenario):
@@ -48,10 +49,19 @@ class MentalHealthScenario(Scenario):
     """
 
     name = "mental_health"
-    description = "A dataset containing a counselor and mental health patient conversation, where the objective is to \
-                    generate an empathetic counselor response."
+    description = (
+        "MentalHealth is a benchmark focused on evaluating empathetic communication in"
+        "mental health counseling. It includes simulated conversations between patients"
+        "and counselors, where the task is to generate compassionate and appropriate counselor"
+        "responses. The benchmark assesses a model's ability to support patients emotionally"
+        "and meaningfully engage in therapeutic conversations."
+    )
     tags = ["dialogue", "counseling", "mental_health", "empathy", "healthcare"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def process_dialogue_data(self, data: pd.DataFrame) -> List[Instance]:
         """
         Process the dialogue data into evaluation instances.
@@ -102,9 +112,10 @@ class MentalHealthScenario(Scenario):
         Returns:
             List[Instance]: List of processed instances for evaluation
         """
-        # Load the processed dialogue data
-        data_path = "/share/pi/nigam/data/medhelm/mental_health/processed_dialogues.csv"
-        dialogue_data = pd.read_csv(data_path)
+        check_file_exists(
+            self.data_path, msg=f"[MentalHealthScenario] Required data file not found: '{self.data_path}'"
+        )
+        dialogue_data = pd.read_csv(self.data_path)
 
         # Process into instances
         instances = self.process_dialogue_data(dialogue_data)

helm/benchmark/scenarios/mimic_bhc_scenario.py

@@ -1,7 +1,7 @@
 import json
 from typing import Dict, List
 
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -14,7 +14,7 @@ from helm.benchmark.scenarios.scenario import (
 
 
 class MIMICBHCScenario(Scenario):
-    """
+    r"""
     MIMIC-IV-BHC presents a curated collection of preprocessed discharge notes with labeled brief hospital
     course (BHC) summaries. This dataset is derived from MIMIC-IV (https://doi.org/10.1093/jamia/ocae312).
 
@@ -59,15 +59,20 @@ class MIMICBHCScenario(Scenario):
 
     name = "mimic_bhc"
     description = (
-        "A summarization task using a curated collection of preprocessed discharge notes"
-        " paired with their corresponding brief hospital course (BHC) summaries."
+        "MIMIC-BHC is a benchmark focused on summarization of discharge notes into Brief"
+        "Hospital Course (BHC) sections. It consists of curated discharge notes from MIMIC-IV,"
+        "each paired with its corresponding BHC summary. The benchmark evaluates a model's"
+        "ability to condense detailed clinical information into accurate, concise summaries that"
+        "reflect the patient's hospital stay."
     )
     tags = ["summarization", "biomedical"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/share/pi/nigam/data/bhc-mimiciv/"
-        ensure_directory_exists(data_path)
-        data_path = data_path + "mimic_iv_bhc.json"
+        check_file_exists(self.data_path, msg=f"[MIMICBHCScenario] Required data file not found: '{self.data_path}'")
 
         instances: List[Instance] = []
         # Limit to zero shot setting for now
@@ -77,7 +82,7 @@ class MIMICBHCScenario(Scenario):
             "test": TEST_SPLIT,
         }
 
-        with open(data_path, "r") as f:
+        with open(self.data_path, "r") as f:
             data = [json.loads(line) for line in f]
 
         for data_split, split in splits.items():

helm/benchmark/scenarios/mimic_rrs_scenario.py

@@ -1,7 +1,7 @@
 import os
 from typing import Dict, List
 
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -44,15 +44,18 @@ class MIMICRRSScenario(Scenario):
 
     name = "mimic_rrs"
     description = (
-        "A dataset containing radiology reports with findings sections from MIMIC-III paired with"
-        " their corresponding impression sections, used for generating radiology report summaries."
+        "MIMIC-RRS is a benchmark constructed from radiology reports in the MIMIC-III"
+        "database. It contains pairs of 'Findings' and 'Impression' sections, enabling evaluation"
+        "of a model's ability to summarize diagnostic imaging observations into concise, clinically"
+        "relevant conclusions."
     )
     tags = ["question_answering", "biomedical"]
 
-    def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/share/pi/nigam/data/rrs-mimiciii/all"
-        ensure_directory_exists(data_path)
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
 
+    def get_instances(self, output_path: str) -> List[Instance]:
         instances: List[Instance] = []
         # Limit to zero shot setting for now
         splits: Dict[str, str] = {
@@ -64,8 +67,14 @@ class MIMICRRSScenario(Scenario):
         for data_split, split in splits.items():
             split_findings_name: str = f"{data_split}.findings.tok"
             split_impressions_name: str = f"{data_split}.impression.tok"
-            findings_path: str = os.path.join(data_path, split_findings_name)
-            impressions_path: str = os.path.join(data_path, split_impressions_name)
+            findings_path: str = os.path.join(self.data_path, split_findings_name)
+            impressions_path: str = os.path.join(self.data_path, split_impressions_name)
+            check_file_exists(
+                findings_path, msg=f"[MIMICRRSScenario] Required findings file not found: '{findings_path}'"
+            )
+            check_file_exists(
+                impressions_path, msg=f"[MIMICRRSScenario] Required impressions file not found: '{impressions_path}'"
+            )
             findings: List[str] = self.read_file(findings_path)
             impressions: List[str] = self.read_file(impressions_path)
             assert len(findings) == len(impressions), "Findings and impressions must have the same length"

helm/benchmark/scenarios/mimiciv_billing_code_scenario.py

@@ -1,9 +1,8 @@
-import os
 import pandas as pd
 import numpy as np
 from typing import List
 
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -24,21 +23,28 @@ class MIMICIVBillingCodeScenario(Scenario):
     """
 
     name = "mimiciv_billing_code"
-    description = "A  dataset pairing clinical notes from MIMIC-IV with corresponding ICD-10 billing codes."
+    description = (
+        "MIMIC-IV Billing Code is a benchmark derived from discharge summaries in the"
+        "MIMIC-IV database, paired with their corresponding ICD-10 billing codes. The task"
+        "requires models to extract structured billing codes based on free-text clinical notes,"
+        "reflecting real-world hospital coding tasks for financial reimbursement."
+    )
     tags = ["question_answering", "biomedical"]
 
-    def __init__(self, data_file: str):
+    def __init__(self, data_path: str):
         """
-        :param data_file: Path to the mimiciv_icd10.feather file.
+        :param data_path: Path to the mimiciv_icd10.feather file.
         """
         super().__init__()
-        self.data_file = data_file
+        self.data_path = data_path
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        ensure_directory_exists(os.path.dirname(self.data_file))
+        check_file_exists(
+            self.data_path, msg=f"[MIMICIVBilligCodeScenario] Required data file not found: '{self.data_path}'"
+        )
 
         # Read the preprocessed MIMIC-IV data (.feather format)
-        df = pd.read_feather(self.data_file)  # columns: ["text", "target", ...]
+        df = pd.read_feather(self.data_path)  # columns: ["text", "target", ...]
 
         instances: List[Instance] = []
 

helm/benchmark/scenarios/mmlu_pro_scenario.py

@@ -40,7 +40,7 @@ class MMLUProScenario(Scenario):
 
     def __init__(self, subject: str):
         super().__init__()
-        self.subject: str = subject
+        self.subject: str = subject.replace("_", " ")
 
     def process_dataset(self, data: Dataset, split: str) -> List[Instance]:
         """

helm/benchmark/scenarios/mmmlu_scenario.py

@@ -0,0 +1,85 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MMMLUScenario(Scenario):
+    """Multilingual Massive Multitask Language Understanding (MMMLU) by OpenAI
+
+    The MMLU is a widely recognized benchmark of general knowledge attained
+    by AI models. It covers a broad range of topics from 57 different categories,
+    covering elementary-level knowledge up to advanced professional subjects like
+    law, physics, history, and computer science.
+
+    MMMLU is a translation of MMLU’s test set into 14 languages using professional
+    human translators. Relying on human translators for this evaluation increases
+    confidence in the accuracy of the translations, especially for low-resource
+    languages like Yoruba.
+
+    The Massive Multitask Language Understanding benchmark from this paper:
+
+    - https://arxiv.org/pdf/2009.03300.pdf
+
+    The MMMLU dataset is from here:
+
+    - https://huggingface.co/datasets/openai/MMMLU
+    """
+
+    name = "mmmlu"
+    description = "Multilingual Massive Multitask Language Understanding"
+    tags = ["knowledge", "multiple_choice"]
+
+    OPTIONS = ["A", "B", "C", "D"]
+
+    def __init__(self, locale: str, subject: str):
+        super().__init__()
+        self.locale: str = locale
+        self.subject: str = subject
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "openai/MMMLU",
+            self.locale,
+            revision="325a01dc3e173cac1578df94120499aaca2e2504",
+            cache_dir=cache_dir,
+            split="test",
+        )
+        assert isinstance(dataset, datasets.Dataset)
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row_index, row in enumerate(dataset):
+            if self.subject != "all" and row["Subject"] != self.subject:
+                continue
+            input = Input(text=row["Question"])
+            references: List[Reference] = []
+            for option in self.OPTIONS:
+                references.append(
+                    Reference(
+                        output=Output(text=row[option]),
+                        tags=[CORRECT_TAG] if option == row["Answer"] else [],
+                    )
+                )
+            instance = Instance(
+                id=f"id{row_index}",
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/mtsamples_procedures_scenario.py

@@ -31,9 +31,12 @@ class MTSamplesProceduresScenario(Scenario):
     GITHUB_DIR_URL = f"https://github.com/raulista1997/benchmarkdata/tree/{GIT_HASH}/mtsample_procedure"
     RAW_BASE_URL = f"https://raw.githubusercontent.com/raulista1997/benchmarkdata/{GIT_HASH}/mtsample_procedure/"
 
-    name = "mtsamples"
+    name = "mtsamples_procedures"
     description = (
-        "A dataset that provides a patient note regarding an operation, with the objective to document the procedure."
+        "MTSamples Procedures is a benchmark composed of transcribed operative notes,"
+        "focused on documenting surgical procedures. Each example presents a brief patient case"
+        "involving a surgical intervention, and the model is tasked with generating a coherent"
+        "and clinically accurate procedural summary or treatment plan."
     )
     tags = ["medical", "transcription", "plan_generation"]
 

helm/benchmark/scenarios/mtsamples_replicate_scenario.py

@@ -36,8 +36,9 @@ class MTSamplesReplicateScenario(Scenario):
 
     name = "mtsamples_replicate"
     description = (
-        "A dataset of clinical notes where the model is prompted to generate "
-        "a reasonable treatment plan for the patient based on transcribed medical reports."
+        "MTSamples Replicate is a benchmark that provides transcribed medical reports"
+        "from various specialties. It is used to evaluate a model's ability to generate clinically"
+        "appropriate treatment plans based on unstructured patient documentation"
     )
     tags = ["medical", "transcription", "plan_generation"]
 

helm/benchmark/scenarios/n2c2_ct_matching_scenario.py

@@ -188,8 +188,13 @@ class N2C2CTMatchingScenario(Scenario):
     """
 
     name = "n2c2_ct_matching"
-    description = "A dataset that provides clinical notes and asks the model to classify whether the \
-    patient is a valid candidate for a provided clinical trial."
+    description = (
+        "N2C2-CT is a benchmark designed to evaluate a model's ability to match patients to"
+        "appropriate clinical trials based on eligibility criteria. Each example includes a clinical"
+        "note and a trial description. The model is tasked with determining whether the patient"
+        "is a valid candidate for the trial. This benchmark supports automation and decision"
+        "support in clinical research enrollment."
+    )
     tags = []  # TODO
 
     POSSIBLE_ANSWER_CHOICES: List[str] = [
@@ -197,11 +202,12 @@ class N2C2CTMatchingScenario(Scenario):
         "no",
     ]
 
-    def __init__(self, subject: str):
+    def __init__(self, data_path: str, subject: str):
         super().__init__()
         self.subject: str = subject  # specific inclusion criterion to assess
-        self.path_to_train_dir: str = "/share/pi/nigam/data/medhelm/n2c2_ct_matching/train/"
-        self.path_to_test_dir: str = "/share/pi/nigam/data/medhelm/n2c2_ct_matching/test/"
+        self.data_path: str = data_path
+        self.path_to_train_dir: str = os.path.join(self.data_path, "train/")
+        self.path_to_test_dir: str = os.path.join(self.data_path, "test/")
 
     def create_prompt(self, patient: Dict[str, Any]) -> str:
         # Cast None values to empty strings during string formatting, but keep the original functions returning None

helm/benchmark/scenarios/openai_mrcr_scenario.py

@@ -0,0 +1,79 @@
+import json
+import os
+import re
+from typing import List, Optional
+
+import datasets
+import tiktoken
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Output,
+    Reference,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    Input,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class OpenAIMRCRScenario(Scenario):
+    """OpenAI MRCR scenario
+
+    OpenAI MRCR (Multi-round co-reference resolution) is a long context dataset for benchmarking
+    an LLM's ability to distinguish between multiple needles hidden in context. This eval is
+    inspired by the MRCR eval first introduced by Gemini (https://arxiv.org/pdf/2409.12640v2).
+
+    The task is as follows: The model is given a long, multi-turn, synthetically generated
+    conversation between user and model where the user asks for a piece of writing about a topic,
+    e.g. "write a poem about tapirs" or "write a blog post about rocks". Hidden in this conversation
+    are 2, 4, or 8 identical asks, and the model is ultimately prompted to return the i-th instance
+    of one of those asks. For example, "Return the 2nd poem about tapirs".
+
+    Reference: https://huggingface.co/datasets/openai/mrcr"""
+
+    name = "openai_mrcr"
+    description = "OpenAI MRCR (Multi-round co-reference resolution) is a long context dataset for benchmarking an LLM's ability to distinguish between multiple needles hidden in context. This eval is inspired by the MRCR eval first introduced by [Vodrahalli et al., 2024](https://arxiv.org/pdf/2409.12640v2)."  # noqa: E501
+    tags = ["long_context", "mrcr"]
+
+    NEEDLES_OPTIONS = [2, 4, 8]
+
+    def __init__(self, needles: int, max_num_words: Optional[int] = None):
+        super().__init__()
+        self.needles = needles
+        self.max_num_words = max_num_words
+        if needles not in self.NEEDLES_OPTIONS:
+            raise Exception(f"Needles must be one of {self.NEEDLES_OPTIONS}")
+        self.tokenizer = tiktoken.get_encoding("o200k_base")
+
+    def count_words(self, messages: list[dict]) -> int:
+        return sum([len(re.split(r"\s+", m["content"].strip())) for m in messages])
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "openai/mrcr",
+            cache_dir=cache_dir,
+            split="train",
+            data_files=[f"{self.needles}needle.parquet"],
+            revision="204b0d4e8d9ca5c0a90bf942fdb2a5969094adc0",
+        )
+        instances = []
+        for idx, row in enumerate(dataset):
+            messages = json.loads(row["prompt"])
+            if self.max_num_words and self.count_words(messages) > self.max_num_words:
+                continue
+            input = Input(messages=messages)
+            references = [Reference(output=Output(text=row["answer"]), tags=[CORRECT_TAG])]
+            instance = Instance(
+                id=f"{self.needles}needle{idx}",
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+                extra_data={"random_string_to_prepend": row["random_string_to_prepend"]},
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/pubmed_qa_scenario.py

@@ -125,7 +125,12 @@ class PubMedQAScenario(Scenario):
     """
 
     name = "pubmed_qa"
-    description = "A dataset that provides pubmed abstracts and asks associated questions yes/no/maybe questions."
+    description = (
+        "PubMedQA is a biomedical question-answering dataset that evaluates a model's"
+        "ability to interpret scientific literature. It consists of PubMed abstracts paired with"
+        "yes/no/maybe questions derived from the content. The benchmark assesses a model's"
+        "capability to reason over biomedical texts and provide factually grounded answers."
+    )
     tags = ["question_answering", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["yes", "no", "maybe"]

helm/benchmark/scenarios/race_based_med_scenario.py

@@ -1,6 +1,6 @@
 import csv
+import os
 
-from filelock import FileLock
 from typing import Dict, List
 from docx import Document
 
@@ -13,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
 )
+from helm.common.general import ensure_file_downloaded
 
 
 def extract_red_text_runs(document):
@@ -87,10 +88,19 @@ class RaceBasedMedScenario(Scenario):
     """
 
     name = "race_based_med"
-    description = "A collection of LLM outputs in response to medical questions with race-based biases, \
-    with the objective being to classify whether the output contains racially biased content."
+    description = (
+        "RaceBias is a benchmark used to evaluate language models for racially biased or"
+        "inappropriate content in medical question-answering scenarios. Each instance consists"
+        "of a medical question and a model-generated response. The task is to classify whether"
+        "the response contains race-based, harmful, or inaccurate content. This benchmark"
+        "supports research into bias detection and fairness in clinical AI systems."
+    )
     tags = ["knowledge", "reasoning", "biomedical"]
     POSSIBLE_ANSWER_CHOICES: List[str] = ["yes", "no"]
+    FILE_URL: str = (
+        "https://static-content.springer.com/esm/"
+        "art%3A10.1038%2Fs41746-023-00939-z/MediaObjects/41746_2023_939_MOESM1_ESM.docx"
+    )
 
     def create_benchmark(self, csv_path) -> Dict[str, str]:
         data = {}
@@ -114,12 +124,12 @@ class RaceBasedMedScenario(Scenario):
         return data
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/share/pi/nigam/data/medhelm/race_based/race_based.csv"
         # Path to the word file from supplement: https://www.nature.com/articles/s41746-023-00939-z#Sec3
-        word_file = "/share/pi/nigam/data/medhelm/race_based/race_based.docx"
-        lock_path = data_path + ".lock"
-        with FileLock(lock_path):
-            # if not os.path.exists(data_path):
+        data_path = os.path.join(output_path, "race_based.csv")
+
+        if not os.path.exists(data_path):
+            word_file = os.path.join(output_path, "race_based.docx")
+            ensure_file_downloaded(source_url=self.FILE_URL, target_path=word_file, unpack=False)
             create_csv_from_word(word_file, data_path)
 
         instances: List[Instance] = []