crfm-helm 0.5.5__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/scenarios/melt_translation_scenario.py

@@ -0,0 +1,152 @@
+from typing import Any, Dict, List, Optional
+
+from datasets import load_dataset, Dataset
+from helm.common.hierarchical_logger import htrack_block
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TRAIN_SPLIT,
+    TEST_SPLIT,
+    VALID_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class MELTTranslationScenario(Scenario):
+    name = "melt_translation"
+    description = "Machine Translation scenario."
+    tags = ["machine_translation"]
+
+    def __init__(
+        self,
+        dataset_name: str,
+        revision: str,
+        source_language: str,
+        target_language: str,
+        subset: Optional[str] = None,
+        splits: Optional[Dict[str, str]] = None,
+    ):
+        """Initializes the question answering scenario.
+
+        Args:
+            dataset_name: The name of the dataset.
+            revision: The revision of the dataset to use.
+            source_language: The source language to use.
+            target_language: The target language to use.
+            subset: The subset of the dataset to use. Defaults to "".
+            splits: The splits to use for the dataset. Defaults to None.
+        """
+        super().__init__()
+        self.MAX_TRAIN_INSTANCES = 20_000
+        valid_languages = set(["vi", "en"])
+        self.dataset_name = dataset_name
+        self.subset = subset
+        self.revision = revision
+        self.splits = splits
+        self.source_language = source_language
+        self.target_language = target_language
+        if self.source_language not in valid_languages or self.target_language not in valid_languages:
+            raise ValueError("Supported languages: vi, en.")
+        if self.source_language == self.target_language:
+            raise ValueError("The source language and the target language should be different.")
+        if self.source_language != "en" and self.target_language != "en":
+            raise ValueError("One of the languages should be English.")
+
+    def get_instances_for_splits(self, splits: Dict[str, str]) -> List[Instance]:
+        """
+        Helper for generating instances for a split.
+        Args:
+            splits (dict): Which splits to partition the data into.
+        Returns:
+            List[Instance]: Instances from the file for the specified split.
+        """
+        with htrack_block("Loading the HuggingFace dataset. The first time could take several minutes."):
+            hf_dataset: Any = load_dataset(
+                self.dataset_name,
+                self.subset,
+                revision=self.revision,
+                trust_remote_code=True,
+            )
+
+        instances: List[Instance] = []
+
+        for dataset_split_name, helm_split_name in splits.items():
+            if helm_split_name == TRAIN_SPLIT:
+                hf_dataset[dataset_split_name] = hf_dataset[dataset_split_name].shuffle(seed=42)[
+                    : self.MAX_TRAIN_INSTANCES
+                ]
+                hf_dataset[dataset_split_name] = Dataset.from_dict(hf_dataset[dataset_split_name])
+
+            for example in hf_dataset[dataset_split_name]:
+                source_sentence = example[self.source_language]
+                target_sentence = example[self.target_language]
+                instances.append(
+                    Instance(
+                        input=Input(text=source_sentence),
+                        references=[Reference(Output(text=target_sentence), tags=[CORRECT_TAG])],
+                        split=helm_split_name,
+                    )
+                )
+        return instances
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        if self.splits is None:
+            splits = {"train": TRAIN_SPLIT, "validation": VALID_SPLIT, "test": TEST_SPLIT}
+        else:
+            splits = {}
+            if "train" in self.splits:
+                splits[self.splits[TRAIN_SPLIT]] = TRAIN_SPLIT
+            if "validation" in self.splits:
+                splits[self.splits[VALID_SPLIT]] = VALID_SPLIT
+            if "test" in self.splits:
+                splits[self.splits[TEST_SPLIT]] = TEST_SPLIT
+
+        instances: List[Instance] = self.get_instances_for_splits(splits=splits)
+        return instances
+
+
+class MELTTranslationOPUS100Scenario(MELTTranslationScenario):
+    """
+    Scenario for the OPUS100 dataset.
+    """
+
+    name = "melt_translation_opus100"
+    description = "OPUS100 dataset for machine translation."
+    tags = ["machine_translation"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="vietgpt/opus100_envi",
+            revision="45df06fb0b31edc882d7c8d34389261f995e5208",
+            splits={
+                TRAIN_SPLIT: "train",
+                VALID_SPLIT: "validation",
+                TEST_SPLIT: "test",
+            },
+            **kwargs,
+        )
+
+
+class MELTTranslationPhoMTScenario(MELTTranslationScenario):
+    """
+    Scenario for the PhoMT dataset.
+    """
+
+    name = "melt_translation_phomt"
+    description = "PhoMT dataset for machine translation."
+    tags = ["machine_translation"]
+
+    def __init__(self, **kwargs):
+        super().__init__(
+            dataset_name="ura-hcmut/PhoMT",
+            revision="74386685db01dc038860ff0a90d9f5fbde284bf7",
+            splits={
+                TRAIN_SPLIT: "train",
+                VALID_SPLIT: "validation",
+                TEST_SPLIT: "test",
+            },
+            **kwargs,
+        )

helm/benchmark/scenarios/mental_health_scenario.py

@@ -9,6 +9,7 @@ from helm.benchmark.scenarios.scenario import (
     PassageQuestionInput,
     Output,
 )
+from helm.common.general import check_file_exists
 
 
 class MentalHealthScenario(Scenario):
@@ -48,10 +49,19 @@ class MentalHealthScenario(Scenario):
     """
 
     name = "mental_health"
-    description = "A dataset containing a counselor and mental health patient conversation, where the objective is to \
-                    generate an empathetic counselor response."
+    description = (
+        "MentalHealth is a benchmark focused on evaluating empathetic communication in"
+        "mental health counseling. It includes simulated conversations between patients"
+        "and counselors, where the task is to generate compassionate and appropriate counselor"
+        "responses. The benchmark assesses a model's ability to support patients emotionally"
+        "and meaningfully engage in therapeutic conversations."
+    )
     tags = ["dialogue", "counseling", "mental_health", "empathy", "healthcare"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def process_dialogue_data(self, data: pd.DataFrame) -> List[Instance]:
         """
         Process the dialogue data into evaluation instances.
@@ -102,9 +112,10 @@ class MentalHealthScenario(Scenario):
         Returns:
             List[Instance]: List of processed instances for evaluation
         """
-        # Load the processed dialogue data
-        data_path = "/share/pi/nigam/data/medhelm/mental_health/processed_dialogues.csv"
-        dialogue_data = pd.read_csv(data_path)
+        check_file_exists(
+            self.data_path, msg=f"[MentalHealthScenario] Required data file not found: '{self.data_path}'"
+        )
+        dialogue_data = pd.read_csv(self.data_path)
 
         # Process into instances
         instances = self.process_dialogue_data(dialogue_data)

helm/benchmark/scenarios/mimic_bhc_scenario.py

@@ -1,7 +1,7 @@
 import json
 from typing import Dict, List
 
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -59,15 +59,20 @@ class MIMICBHCScenario(Scenario):
 
     name = "mimic_bhc"
     description = (
-        "A summarization task using a curated collection of preprocessed discharge notes"
-        " paired with their corresponding brief hospital course (BHC) summaries."
+        "MIMIC-BHC is a benchmark focused on summarization of discharge notes into Brief"
+        "Hospital Course (BHC) sections. It consists of curated discharge notes from MIMIC-IV,"
+        "each paired with its corresponding BHC summary. The benchmark evaluates a model's"
+        "ability to condense detailed clinical information into accurate, concise summaries that"
+        "reflect the patient's hospital stay."
     )
     tags = ["summarization", "biomedical"]
 
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
+
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/share/pi/nigam/data/bhc-mimiciv/"
-        ensure_directory_exists(data_path)
-        data_path = data_path + "mimic_iv_bhc.json"
+        check_file_exists(self.data_path, msg=f"[MIMICBHCScenario] Required data file not found: '{self.data_path}'")
 
         instances: List[Instance] = []
         # Limit to zero shot setting for now
@@ -77,7 +82,7 @@ class MIMICBHCScenario(Scenario):
             "test": TEST_SPLIT,
         }
 
-        with open(data_path, "r") as f:
+        with open(self.data_path, "r") as f:
             data = [json.loads(line) for line in f]
 
         for data_split, split in splits.items():

helm/benchmark/scenarios/mimic_rrs_scenario.py

@@ -1,7 +1,7 @@
 import os
 from typing import Dict, List
 
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -44,15 +44,18 @@ class MIMICRRSScenario(Scenario):
 
     name = "mimic_rrs"
     description = (
-        "A dataset containing radiology reports with findings sections from MIMIC-III paired with"
-        " their corresponding impression sections, used for generating radiology report summaries."
+        "MIMIC-RRS is a benchmark constructed from radiology reports in the MIMIC-III"
+        "database. It contains pairs of 'Findings' and 'Impression' sections, enabling evaluation"
+        "of a model's ability to summarize diagnostic imaging observations into concise, clinically"
+        "relevant conclusions."
     )
     tags = ["question_answering", "biomedical"]
 
-    def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/share/pi/nigam/data/rrs-mimiciii/all"
-        ensure_directory_exists(data_path)
+    def __init__(self, data_path: str):
+        super().__init__()
+        self.data_path = data_path
 
+    def get_instances(self, output_path: str) -> List[Instance]:
         instances: List[Instance] = []
         # Limit to zero shot setting for now
         splits: Dict[str, str] = {
@@ -64,8 +67,14 @@ class MIMICRRSScenario(Scenario):
         for data_split, split in splits.items():
             split_findings_name: str = f"{data_split}.findings.tok"
             split_impressions_name: str = f"{data_split}.impression.tok"
-            findings_path: str = os.path.join(data_path, split_findings_name)
-            impressions_path: str = os.path.join(data_path, split_impressions_name)
+            findings_path: str = os.path.join(self.data_path, split_findings_name)
+            impressions_path: str = os.path.join(self.data_path, split_impressions_name)
+            check_file_exists(
+                findings_path, msg=f"[MIMICRRSScenario] Required findings file not found: '{findings_path}'"
+            )
+            check_file_exists(
+                impressions_path, msg=f"[MIMICRRSScenario] Required impressions file not found: '{impressions_path}'"
+            )
             findings: List[str] = self.read_file(findings_path)
             impressions: List[str] = self.read_file(impressions_path)
             assert len(findings) == len(impressions), "Findings and impressions must have the same length"

helm/benchmark/scenarios/mimiciv_billing_code_scenario.py

@@ -1,9 +1,8 @@
-import os
 import pandas as pd
 import numpy as np
 from typing import List
 
-from helm.common.general import ensure_directory_exists
+from helm.common.general import check_file_exists
 from helm.benchmark.scenarios.scenario import (
     Input,
     Scenario,
@@ -24,21 +23,28 @@ class MIMICIVBillingCodeScenario(Scenario):
     """
 
     name = "mimiciv_billing_code"
-    description = "A  dataset pairing clinical notes from MIMIC-IV with corresponding ICD-10 billing codes."
+    description = (
+        "MIMIC-IV Billing Code is a benchmark derived from discharge summaries in the"
+        "MIMIC-IV database, paired with their corresponding ICD-10 billing codes. The task"
+        "requires models to extract structured billing codes based on free-text clinical notes,"
+        "reflecting real-world hospital coding tasks for financial reimbursement."
+    )
     tags = ["question_answering", "biomedical"]
 
-    def __init__(self, data_file: str):
+    def __init__(self, data_path: str):
         """
-        :param data_file: Path to the mimiciv_icd10.feather file.
+        :param data_path: Path to the mimiciv_icd10.feather file.
         """
         super().__init__()
-        self.data_file = data_file
+        self.data_path = data_path
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        ensure_directory_exists(os.path.dirname(self.data_file))
+        check_file_exists(
+            self.data_path, msg=f"[MIMICIVBilligCodeScenario] Required data file not found: '{self.data_path}'"
+        )
 
         # Read the preprocessed MIMIC-IV data (.feather format)
-        df = pd.read_feather(self.data_file)  # columns: ["text", "target", ...]
+        df = pd.read_feather(self.data_path)  # columns: ["text", "target", ...]
 
         instances: List[Instance] = []
 

helm/benchmark/scenarios/mmlu_pro_scenario.py

@@ -40,7 +40,7 @@ class MMLUProScenario(Scenario):
 
     def __init__(self, subject: str):
         super().__init__()
-        self.subject: str = subject
+        self.subject: str = subject.replace("_", " ")
 
     def process_dataset(self, data: Dataset, split: str) -> List[Instance]:
         """

helm/benchmark/scenarios/mtsamples_procedures_scenario.py

@@ -31,9 +31,12 @@ class MTSamplesProceduresScenario(Scenario):
     GITHUB_DIR_URL = f"https://github.com/raulista1997/benchmarkdata/tree/{GIT_HASH}/mtsample_procedure"
     RAW_BASE_URL = f"https://raw.githubusercontent.com/raulista1997/benchmarkdata/{GIT_HASH}/mtsample_procedure/"
 
-    name = "mtsamples"
+    name = "mtsamples_procedures"
     description = (
-        "A dataset that provides a patient note regarding an operation, with the objective to document the procedure."
+        "MTSamples Procedures is a benchmark composed of transcribed operative notes,"
+        "focused on documenting surgical procedures. Each example presents a brief patient case"
+        "involving a surgical intervention, and the model is tasked with generating a coherent"
+        "and clinically accurate procedural summary or treatment plan."
     )
     tags = ["medical", "transcription", "plan_generation"]
 

helm/benchmark/scenarios/mtsamples_replicate_scenario.py

@@ -36,8 +36,9 @@ class MTSamplesReplicateScenario(Scenario):
 
     name = "mtsamples_replicate"
     description = (
-        "A dataset of clinical notes where the model is prompted to generate "
-        "a reasonable treatment plan for the patient based on transcribed medical reports."
+        "MTSamples Replicate is a benchmark that provides transcribed medical reports"
+        "from various specialties. It is used to evaluate a model's ability to generate clinically"
+        "appropriate treatment plans based on unstructured patient documentation"
     )
     tags = ["medical", "transcription", "plan_generation"]
 

helm/benchmark/scenarios/n2c2_ct_matching_scenario.py

@@ -188,8 +188,13 @@ class N2C2CTMatchingScenario(Scenario):
     """
 
     name = "n2c2_ct_matching"
-    description = "A dataset that provides clinical notes and asks the model to classify whether the \
-    patient is a valid candidate for a provided clinical trial."
+    description = (
+        "N2C2-CT is a benchmark designed to evaluate a model's ability to match patients to"
+        "appropriate clinical trials based on eligibility criteria. Each example includes a clinical"
+        "note and a trial description. The model is tasked with determining whether the patient"
+        "is a valid candidate for the trial. This benchmark supports automation and decision"
+        "support in clinical research enrollment."
+    )
     tags = []  # TODO
 
     POSSIBLE_ANSWER_CHOICES: List[str] = [
@@ -197,11 +202,12 @@ class N2C2CTMatchingScenario(Scenario):
         "no",
     ]
 
-    def __init__(self, subject: str):
+    def __init__(self, data_path: str, subject: str):
         super().__init__()
         self.subject: str = subject  # specific inclusion criterion to assess
-        self.path_to_train_dir: str = "/share/pi/nigam/data/medhelm/n2c2_ct_matching/train/"
-        self.path_to_test_dir: str = "/share/pi/nigam/data/medhelm/n2c2_ct_matching/test/"
+        self.data_path: str = data_path
+        self.path_to_train_dir: str = os.path.join(self.data_path, "train/")
+        self.path_to_test_dir: str = os.path.join(self.data_path, "test/")
 
     def create_prompt(self, patient: Dict[str, Any]) -> str:
         # Cast None values to empty strings during string formatting, but keep the original functions returning None

helm/benchmark/scenarios/numeracy_scenario.py

@@ -11,6 +11,7 @@ from typing import List, Optional, Tuple, Dict
 
 from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_GENERATION
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.common.local_context import LocalContext
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
 from helm.common.authentication import Authentication
 from helm.common.optional_dependencies import handle_module_not_found_error
@@ -39,7 +40,7 @@ except ModuleNotFoundError as e:
 #       https://github.com/stanford-crfm/benchmarking/issues/569
 def get_test_tokenizer_service() -> TokenizerService:
     # Pointed to the default local path set in run.py (--local-path)
-    return TokenizerService(ServerService(base_path="prod_env", root_mode=True), Authentication("test"))
+    return TokenizerService(LocalContext(base_path="prod_env"))
 
 
 SOLUTION_TAG: str = "solution"

helm/benchmark/scenarios/openai_mrcr_scenario.py

@@ -0,0 +1,79 @@
+import json
+import os
+import re
+from typing import List, Optional
+
+import datasets
+import tiktoken
+
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+    Output,
+    Reference,
+    Scenario,
+    Instance,
+    TEST_SPLIT,
+    Input,
+)
+from helm.common.general import ensure_directory_exists
+
+
+class OpenAIMRCRScenario(Scenario):
+    """OpenAI MRCR scenario
+
+    OpenAI MRCR (Multi-round co-reference resolution) is a long context dataset for benchmarking
+    an LLM's ability to distinguish between multiple needles hidden in context. This eval is
+    inspired by the MRCR eval first introduced by Gemini (https://arxiv.org/pdf/2409.12640v2).
+
+    The task is as follows: The model is given a long, multi-turn, synthetically generated
+    conversation between user and model where the user asks for a piece of writing about a topic,
+    e.g. "write a poem about tapirs" or "write a blog post about rocks". Hidden in this conversation
+    are 2, 4, or 8 identical asks, and the model is ultimately prompted to return the i-th instance
+    of one of those asks. For example, "Return the 2nd poem about tapirs".
+
+    Reference: https://huggingface.co/datasets/openai/mrcr"""
+
+    name = "openai_mrcr"
+    description = "OpenAI MRCR (Multi-round co-reference resolution) is a long context dataset for benchmarking an LLM's ability to distinguish between multiple needles hidden in context. This eval is inspired by the MRCR eval first introduced by [Vodrahalli et al., 2024](https://arxiv.org/pdf/2409.12640v2)."  # noqa: E501
+    tags = ["long_context", "mrcr"]
+
+    NEEDLES_OPTIONS = [2, 4, 8]
+
+    def __init__(self, needles: int, max_num_words: Optional[int] = None):
+        super().__init__()
+        self.needles = needles
+        self.max_num_words = max_num_words
+        if needles not in self.NEEDLES_OPTIONS:
+            raise Exception(f"Needles must be one of {self.NEEDLES_OPTIONS}")
+        self.tokenizer = tiktoken.get_encoding("o200k_base")
+
+    def count_words(self, messages: list[dict]) -> int:
+        return sum([len(re.split(r"\s+", m["content"].strip())) for m in messages])
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset = datasets.load_dataset(
+            "openai/mrcr",
+            cache_dir=cache_dir,
+            split="train",
+            data_files=[f"{self.needles}needle.parquet"],
+            revision="204b0d4e8d9ca5c0a90bf942fdb2a5969094adc0",
+        )
+        instances = []
+        for idx, row in enumerate(dataset):
+            messages = json.loads(row["prompt"])
+            if self.max_num_words and self.count_words(messages) > self.max_num_words:
+                continue
+            input = Input(messages=messages)
+            references = [Reference(output=Output(text=row["answer"]), tags=[CORRECT_TAG])]
+            instance = Instance(
+                id=f"{self.needles}needle{idx}",
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+                extra_data={"random_string_to_prepend": row["random_string_to_prepend"]},
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/pubmed_qa_scenario.py

@@ -125,7 +125,12 @@ class PubMedQAScenario(Scenario):
     """
 
     name = "pubmed_qa"
-    description = "A dataset that provides pubmed abstracts and asks associated questions yes/no/maybe questions."
+    description = (
+        "PubMedQA is a biomedical question-answering dataset that evaluates a model's"
+        "ability to interpret scientific literature. It consists of PubMed abstracts paired with"
+        "yes/no/maybe questions derived from the content. The benchmark assesses a model's"
+        "capability to reason over biomedical texts and provide factually grounded answers."
+    )
     tags = ["question_answering", "biomedical"]
 
     POSSIBLE_ANSWER_CHOICES: List[str] = ["yes", "no", "maybe"]

helm/benchmark/scenarios/race_based_med_scenario.py

@@ -1,6 +1,6 @@
 import csv
+import os
 
-from filelock import FileLock
 from typing import Dict, List
 from docx import Document
 
@@ -13,6 +13,7 @@ from helm.benchmark.scenarios.scenario import (
     Reference,
     Output,
 )
+from helm.common.general import ensure_file_downloaded
 
 
 def extract_red_text_runs(document):
@@ -87,10 +88,19 @@ class RaceBasedMedScenario(Scenario):
     """
 
     name = "race_based_med"
-    description = "A collection of LLM outputs in response to medical questions with race-based biases, \
-    with the objective being to classify whether the output contains racially biased content."
+    description = (
+        "RaceBias is a benchmark used to evaluate language models for racially biased or"
+        "inappropriate content in medical question-answering scenarios. Each instance consists"
+        "of a medical question and a model-generated response. The task is to classify whether"
+        "the response contains race-based, harmful, or inaccurate content. This benchmark"
+        "supports research into bias detection and fairness in clinical AI systems."
+    )
     tags = ["knowledge", "reasoning", "biomedical"]
     POSSIBLE_ANSWER_CHOICES: List[str] = ["yes", "no"]
+    FILE_URL: str = (
+        "https://static-content.springer.com/esm/"
+        "art%3A10.1038%2Fs41746-023-00939-z/MediaObjects/41746_2023_939_MOESM1_ESM.docx"
+    )
 
     def create_benchmark(self, csv_path) -> Dict[str, str]:
         data = {}
@@ -114,12 +124,12 @@ class RaceBasedMedScenario(Scenario):
         return data
 
     def get_instances(self, output_path: str) -> List[Instance]:
-        data_path = "/share/pi/nigam/data/medhelm/race_based/race_based.csv"
         # Path to the word file from supplement: https://www.nature.com/articles/s41746-023-00939-z#Sec3
-        word_file = "/share/pi/nigam/data/medhelm/race_based/race_based.docx"
-        lock_path = data_path + ".lock"
-        with FileLock(lock_path):
-            # if not os.path.exists(data_path):
+        data_path = os.path.join(output_path, "race_based.csv")
+
+        if not os.path.exists(data_path):
+            word_file = os.path.join(output_path, "race_based.docx")
+            ensure_file_downloaded(source_url=self.FILE_URL, target_path=word_file, unpack=False)
             create_csv_from_word(word_file, data_path)
 
         instances: List[Instance] = []

helm/benchmark/scenarios/ruler_qa_scenario_helper.py

@@ -133,7 +133,7 @@ def generate_samples(dataset: str, dataset_path: str, template: str, random_seed
         input_text, answer = generate_input_output(0, num_docs, template=template, random_seed=random_seed, qas=qas, docs=docs)
         # Calculate the number of tokens in the example
         total_tokens = len(_text_to_tokens(input_text + f' {answer}'))
-        print(f'Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate} | Docs: {num_docs}')
+        # print(f'Max length {max_seq_length} | Current length {total_tokens + tokens_to_generate} | Docs: {num_docs}')
         if total_tokens + tokens_to_generate > max_seq_length:
             num_docs -= incremental
             break
@@ -142,7 +142,7 @@ def generate_samples(dataset: str, dataset_path: str, template: str, random_seed
         if num_docs > len(docs):
             num_docs = len(docs)
             break
-    print('Number of documents:', num_docs)
+    # print('Number of documents:', num_docs)
     
     # Generate samples
     for index in tqdm(range(num_samples)):

helm/benchmark/scenarios/ruler_qa_scenarios.py

@@ -72,7 +72,7 @@ Question: {query} Answer:"""  # noqa: E501
 
 class RULERHotpotQAScenario(_RULERQAScenario):
     name = "ruler_hotpotqa"
-    description = "The HotpotQA long-context multi-hop RAG question answering scenario from RULER"
+    description = "RULER HotPotQA is an augmented version of HotPotQA ([Yang et al., 2018](https://arxiv.org/abs/1809.09600)) introduced by [Hsieh et al., 2024](https://arxiv.org/abs/2404.06654) to simulate a multi-hop question answering as a long-context scenario."  # noqa: E501
     tags = ["long_context", "rag"]
 
     def __init__(self, max_num_words: int):
@@ -81,7 +81,7 @@ class RULERHotpotQAScenario(_RULERQAScenario):
 
 class RULERSQuADScenario(_RULERQAScenario):
     name = "ruler_squad"
-    description = "The SQuAD question answering scenario from RULER"
+    description = "RULER SQuAD is an augmented version of SQuAD ([Rajpurkar et al., 2018](https://arxiv.org/abs/1806.03822)) introduced by [Hsieh et al., 2024](https://arxiv.org/abs/2404.06654) to simulate a single-hop question answering as a long-context scenario."  # noqa: E501
     tags = ["long_context", "rag"]
 
     def __init__(self, max_num_words: int):