crfm-helm 0.5.5__py3-none-any.whl → 0.5.7__py3-none-any.whl

helm/benchmark/run_specs/multilingual_run_specs.py

@@ -0,0 +1,50 @@
+from helm.benchmark.adaptation.adapter_spec import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("mmmlu")
+def get_mmmlu_spec(locale: str, subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.mmmlu_scenario.MMMLUScenario", args={"locale": locale, "subject": subject}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions=f"The following are multiple choice questions (with answers) about {subject.replace('_', ' ')}. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name=f"mmmlu:locale={locale},subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["mmmlu", f"mmmlu_{locale}_{subject}"],
+    )
+
+
+@run_spec_function("exams_multilingual")
+def get_exams_multilingual_spec(language: str, subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.exams_multilingual_scenario.EXAMSMultilingualScenario",
+        args={"language": language, "subject": subject},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions=f"The following are multiple choice questions (with answers) about {subject.replace('_', ' ')}. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name=f"exams_multilingual:locale={language},subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["exams_multilingual", f"exams_multilingual_{language}_{subject}"],
+    )

helm/benchmark/run_specs/speech_disorder_audio_run_specs.py

@@ -0,0 +1,163 @@
+from typing import List, Optional
+from helm.benchmark.adaptation.adapter_spec import (
+    ADAPT_GENERATION_MULTIMODAL,
+    ADAPT_MULTIPLE_CHOICE_JOINT_MULTIMODAL,
+    AdapterSpec,
+)
+from helm.benchmark.metrics.common_metric_specs import (
+    get_basic_generation_metric_specs,
+    get_basic_metric_specs,
+    get_multiple_choice_classification_metric_specs,
+)
+from helm.benchmark.metrics.metric import MetricSpec
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+def audio_classification_metric_specs() -> List[MetricSpec]:
+    return get_multiple_choice_classification_metric_specs() + get_basic_metric_specs(
+        ["exact_match", "quasi_exact_match"]
+    )
+
+
+def _get_multiple_choice_joint_adapter_spec(
+    input_noun: Optional[str],
+    output_noun: str,
+    max_train_instances: int = 0,
+    num_outputs: int = 1,
+) -> AdapterSpec:
+    return AdapterSpec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT_MULTIMODAL,
+        global_prefix="",
+        instructions="Answer the multiple choice question by just giving the letter of the correct answer "
+        "and nothing else.",
+        input_prefix=f"{input_noun}: " if input_noun is not None else "",
+        input_suffix="\n",
+        output_prefix=f"{output_noun}: ",
+        output_suffix="\n",
+        instance_prefix="\n",
+        max_train_instances=max_train_instances,
+        num_outputs=num_outputs,
+        max_tokens=1,
+        stop_sequences=["\n"],
+        temperature=0.0,
+        random=None,
+    )
+
+
+def _get_generation_adapter_spec(
+    max_tokens: int,
+    instructions: str = "",
+    max_train_instances: int = 0,
+    temperature: float = 0.0,
+    stop_sequences: Optional[List[str]] = None,
+) -> AdapterSpec:
+    return AdapterSpec(
+        method=ADAPT_GENERATION_MULTIMODAL,
+        instructions=instructions,
+        input_prefix="",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        instance_prefix="",
+        max_train_instances=max_train_instances,
+        num_outputs=1,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        stop_sequences=stop_sequences if stop_sequences is not None else [],
+    )
+
+
+@run_spec_function("ultra_suite_classification")
+def get_ultra_suite_classification_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_classification_scenario.UltraSuiteClassificationScenario",  # noqa: E501
+    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(input_noun=None, output_noun="Answer")
+    metric_specs: List[MetricSpec] = audio_classification_metric_specs()
+    run_spec_name: str = "ultra_suite_classification"
+    return RunSpec(
+        name=f"{run_spec_name}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+@run_spec_function("ultra_suite_classification_breakdown")
+def get_ultra_suite_disorder_breakdown_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_disorder_breakdown_scenario.UltraSuiteDisorderBreakdownScenario",  # noqa: E501
+    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(input_noun=None, output_noun="Answer")
+    metric_specs: List[MetricSpec] = audio_classification_metric_specs()
+    run_spec_name: str = "ultra_suite_classification_breakdown"
+    return RunSpec(
+        name=f"{run_spec_name}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+# Makes the model transcribe the child's speech into text without assuming what the child is supposed to say
+# if the transcription matches the prompt, then it is classified as typically developing
+# otherwise, it is classified as having a speech disorder
+@run_spec_function("ultra_suite_asr_classification")
+def get_ultra_suite_asr_classification_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_classification_scenario.UltraSuiteASRClassificationScenario",  # noqa: E501
+    )
+    adapter_spec = _get_generation_adapter_spec(
+        instructions="""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording is provided to you, typically consisting of a speech prompt from a pathologist followed by a child's repetition. Based on your expertise transcribe the child's speech into text. Do not make any assumptions about the words the child is expected to say. Only transcribe based on the words that the child actually says. Only respond with the text transcription, no other text or commentary.""",  # noqa: E501
+        max_tokens=10,
+    )
+    metric_specs: List[MetricSpec] = audio_classification_metric_specs()
+    run_spec_name: str = "ultra_suite_asr_classification"
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+# Makes the model transcribe the child's speech into text and is allowed to assume what the child is supposed to say
+@run_spec_function("ultra_suite_asr_transcription")
+def get_ultra_suite_asr_transcription_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_transcription_scenario.UltraSuiteASRTranscriptionScenario",  # noqa: E501
+    )
+    adapter_spec = _get_generation_adapter_spec(
+        instructions="""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. Based on your expertise transcribe the child's speech into text. Try to understand what the child is expected to say. And only respond with the transcription of the child's speech. Not the pathologist's prompt or any other commentary. Only respond with the text transcription, no other text, commentary or punctuations.""",  # noqa: E501
+        max_tokens=50,
+    )
+    metric_specs: List[MetricSpec] = get_basic_generation_metric_specs(["wer_score", "mer_score", "wip_score"])
+    run_spec_name: str = "ultra_suite_asr_transcription"
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+@run_spec_function("ultra_suite_disorder_symptoms")
+def get_ultra_suite_disorder_symptoms_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_disorder_symptoms_scenario.UltraSuiteDisorderSymptomsScenario",  # noqa: E501
+    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(input_noun=None, output_noun="Answer")
+    metric_specs: List[MetricSpec] = audio_classification_metric_specs()
+    run_spec_name: str = "ultra_suite_disorder_symptoms"
+    return RunSpec(
+        name=f"{run_spec_name}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )

helm/benchmark/run_specs/vlm_run_specs.py

@@ -1027,3 +1027,31 @@ def get_vqa_rad_spec() -> RunSpec:
         metric_specs=metric_specs,
         groups=[run_spec_name],
     )
+
+
+############################################################
+# Video understanding run specs
+
+
+@run_spec_function("msr_vtt")
+def get_msr_vtt_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.vision_language.msr_vtt_scenario.MSRVTTScenario",
+        args={},
+    )
+
+    adapter_spec: AdapterSpec = _get_generation_adapter_spec(
+        instructions="Generate a short caption for the video in plain words. Just give the caption and nothing else.",
+        max_tokens=30,
+        max_train_instances=0,
+    )
+    metric_specs: List[MetricSpec] = _get_open_ended_generation_metric_specs()
+
+    run_spec_name: str = "msr_vtt"
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )

helm/benchmark/runner.py

@@ -13,7 +13,7 @@ from tqdm import tqdm
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.common.general import ensure_directory_exists, write, asdict_without_nones
-from helm.common.hierarchical_logger import hlog, htrack_block
+from helm.common.hierarchical_logger import hlog, htrack_block, hwarn
 from helm.common.cache import cache_stats
 from helm.benchmark.scenarios.scenario import (
     EVAL_SPLITS,
@@ -82,7 +82,7 @@ def remove_stats_nans(stats: List[Stat]) -> List[Stat]:
     result: List[Stat] = []
     for stat in stats:
         if math.isnan(stat.sum):
-            hlog(f"WARNING: Removing stat {stat.name.name} because its value is NaN")
+            hwarn(f"Removing stat {stat.name.name} because its value is NaN")
             continue
         result.append(stat)
     return result
@@ -164,8 +164,8 @@ class Runner:
             )
         )
         self.dry_run: bool = execution_spec.dry_run
-        self.tokenizer_service = TokenizerService(self.executor.service, execution_spec.auth)
-        self.metric_service = MetricService(self.executor.service, execution_spec.auth)
+        self.tokenizer_service = TokenizerService(self.executor.context)
+        self.metric_service = MetricService(self.executor.context)
         self.skip_instances: bool = skip_instances
         self.cache_instances: bool = cache_instances
         self.cache_instances_only: bool = cache_instances_only
@@ -318,7 +318,7 @@ class Runner:
         metric_counts: typing.Counter[MetricName] = Counter([stat.name for stat in stats])
         for metric_name, count in metric_counts.items():
             if count > 1:
-                hlog(f"WARNING: duplicate metric name {metric_name}")
+                hwarn(f"duplicate metric name {metric_name}")
 
         # Print out the number of stats
         hlog(f"Generated {len(stats)} stats.")

helm/benchmark/scenarios/aci_bench_scenario.py

@@ -72,7 +72,13 @@ class ACIBenchScenario(Scenario):
     ]
 
     name = "aci_bench"
-    description = "A dataset of patient-doctor conversations paired with structured clinical notes."
+    description = (
+        "ACI-Bench is a benchmark of real-world patient-doctor conversations paired with"
+        "structured clinical notes. The benchmark evaluates a model's ability to understand"
+        "spoken medical dialogue and convert it into formal clinical documentation, covering"
+        "sections such as history of present illness, physical exam findings, results, and assessment"
+        "and plan."
+    )
     tags = ["summarization", "medicine"]
 
     def download_json(self, url: str, output_path: str, file_name: str) -> str:

helm/benchmark/scenarios/alghafa_scenario.py

@@ -0,0 +1,126 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class AlGhafaScenario(Scenario):
+    """AlGhafa Evaluation Benchmark for Arabic Language Models
+
+    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
+
+    Multiple-choice evaluation benchmark for zero- and few-shot evaluation of Arabic LLMs,
+    consisting of
+
+    - https://huggingface.co/datasets/OALL/AlGhafa-Arabic-LLM-Benchmark-Native/
+    - https://aclanthology.org/2023.arabicnlp-1.21/
+
+    Citation:
+
+    ```
+    @inproceedings{almazrouei-etal-2023-alghafa,
+        title = "{A}l{G}hafa Evaluation Benchmark for {A}rabic Language Models",
+        author = "Almazrouei, Ebtesam  and
+        Cojocaru, Ruxandra  and
+        Baldo, Michele  and
+        Malartic, Quentin  and
+        Alobeidli, Hamza  and
+        Mazzotta, Daniele  and
+        Penedo, Guilherme  and
+        Campesan, Giulia  and
+        Farooq, Mugariya  and
+        Alhammadi, Maitha  and
+        Launay, Julien  and
+        Noune, Badreddine",
+        editor = "Sawaf, Hassan  and
+        El-Beltagy, Samhaa  and
+        Zaghouani, Wajdi  and
+        Magdy, Walid  and
+        Abdelali, Ahmed  and
+        Tomeh, Nadi  and
+        Abu Farha, Ibrahim  and
+        Habash, Nizar  and
+        Khalifa, Salam  and
+        Keleg, Amr  and
+        Haddad, Hatem  and
+        Zitouni, Imed  and
+        Mrini, Khalil  and
+        Almatham, Rawan",
+        booktitle = "Proceedings of ArabicNLP 2023",
+        month = dec,
+        year = "2023",
+        address = "Singapore (Hybrid)",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2023.arabicnlp-1.21/",
+        doi = "10.18653/v1/2023.arabicnlp-1.21",
+        pages = "244--275",
+        abstract = "Recent advances in the space of Arabic large language models have opened up a wealth of potential practical applications. From optimal training strategies, large scale data acquisition and continuously increasing NLP resources, the Arabic LLM landscape has improved in a very short span of time, despite being plagued by training data scarcity and limited evaluation resources compared to English. In line with contributing towards this ever-growing field, we introduce AlGhafa, a new multiple-choice evaluation benchmark for Arabic LLMs. For showcasing purposes, we train a new suite of models, including a 14 billion parameter model, the largest monolingual Arabic decoder-only model to date. We use a collection of publicly available datasets, as well as a newly introduced HandMade dataset consisting of 8 billion tokens. Finally, we explore the quantitative and qualitative toxicity of several Arabic models, comparing our models to existing public Arabic LLMs."
+    }
+    ```
+    """  # noqa: E501
+
+    name = "alghafa"
+    description = "AlGhafa"
+    tags = ["multiple choice"]
+
+    HF_SPLIT_TO_HELM_SPLIT = {"validation": TRAIN_SPLIT, "test": TEST_SPLIT}
+    REFERENCE_PREFIX = "sol"
+
+    def __init__(self, subset: str):
+        super().__init__()
+        self.subset = subset
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
+            self.subset,
+            revision="a31ebd34ca311d7e0cfc6ad7f458b3435af280f5",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            option_indexes = [
+                int(s.removeprefix(self.REFERENCE_PREFIX))
+                for s in dataset[0].keys()
+                if s.startswith(self.REFERENCE_PREFIX)
+            ]
+            for row_index, row in enumerate(dataset):
+                input = Input(text=row["query"])
+                references: List[Reference] = []
+                # Need to add 1 because label is zero-indexed and has a value from 0 to (N - 1),
+                # but column names are 1 indexed and have values from "sol1" to "solN"
+                correct_option_index = int(row["label"]) + 1
+                for option_index in option_indexes:
+                    column_name = f"{self.REFERENCE_PREFIX}{option_index}"
+                    references.append(
+                        Reference(
+                            output=Output(text=row[column_name]),
+                            tags=[CORRECT_TAG] if option_index == correct_option_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=f"id{row_index}_{split_name}",
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/arabic_mmlu_scenario.py

@@ -0,0 +1,78 @@
+import os
+from typing import Dict, List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class ArabicMMLUScenario(Scenario):
+    """ArabicMMLU
+
+    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
+
+    ArabicMMLU is the first multi-task language understanding benchmark
+    for Arabic language, sourced from school exams across diverse educational
+    levels in different countries spanning North Africa, the Levant, and the
+    Gulf regions. The data comprises 40 tasks and 14,575 multiple-choice questions
+    in Modern Standard Arabic (MSA), and is carefully constructed by collaborating
+    with native speakers in the region.
+
+    - https://huggingface.co/datasets/MBZUAI/ArabicMMLU
+    - https://aclanthology.org/2024.findings-acl.334/
+    """
+
+    name = "arabic_mmlu"
+    description = "Arabic Massive Multitask Language Understanding"
+    tags = ["knowledge", "multiple_choice"]
+
+    OPTIONS = ["A", "B", "C", "D"]
+    HF_SPLIT_TO_HELM_SPLIT = {"dev": TRAIN_SPLIT, "test": TEST_SPLIT}
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
+            "MBZUAI/ArabicMMLU",
+            "All",
+            revision="7aa530e2893ac420352b3f5c1a1310c010e9758b",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            assert isinstance(dataset, datasets.Dataset)
+            for row_index, row in enumerate(dataset):
+                input = Input(text=row["Question"])
+                references: List[Reference] = []
+                correct_option_index = ord(row["Answer Key"]) - ord("A") + 1
+                for option_index in range(1, 6):
+                    column_name = f"Option {option_index}"
+                    if not row[column_name]:
+                        continue
+                    references.append(
+                        Reference(
+                            output=Output(text=row[column_name]),
+                            tags=[CORRECT_TAG] if option_index == correct_option_index else [],
+                        )
+                    )
+                instance = Instance(
+                    id=f"id{row_index}",
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/aratrust_scenario.py

@@ -0,0 +1,76 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class AraTrustScenario(Scenario):
+    """AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic
+
+    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
+
+    AraTrust is a comprehensive Trustworthiness benchmark for LLMs in Arabic.
+    AraTrust comprises 522 human-written multiple-choice questions addressing
+    diverse dimensions related to truthfulness, ethics, safety, physical health,
+    mental health, unfairness, illegal activities, privacy, and offensive language.
+
+    - https://huggingface.co/datasets/asas-ai/AraTrust
+    - https://arxiv.org/abs/2403.09017
+
+    Citation:
+
+    ```
+    @misc{alghamdi2024aratrustevaluationtrustworthinessllms,
+      title={AraTrust: An Evaluation of Trustworthiness for LLMs in Arabic},
+      author={Emad A. Alghamdi and Reem I. Masoud and Deema Alnuhait and Afnan Y. Alomairi and Ahmed Ashraf and Mohamed Zaytoon},
+      year={2024},
+      eprint={2403.09017},
+      archivePrefix={arXiv},
+      primaryClass={cs.CL},
+      url={https://arxiv.org/abs/2403.09017},
+    }
+    ```
+    """  # noqa: E501
+
+    name = "aratrust"
+    description = "aratrust"
+    tags = ["trustworthiness"]
+
+    OPTION_KEYS = ["A", "B", "C"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset: datasets.Dataset = datasets.load_dataset(
+            "asas-ai/AraTrust",
+            revision="d4dd124ed5b90aeb65a7dda7d88e34fb464a31ec",
+            cache_dir=cache_dir,
+            split="test",
+        )
+        instances: List[Instance] = []
+        for row_index, row in enumerate(dataset):
+            question_text = row["Question"]
+            option_texts = [row[option_key] for option_key in self.OPTION_KEYS if row[option_key]]
+            joined_option_texts = "\n".join(option_texts)
+            input = Input(text=f"{question_text}\n\n{joined_option_texts}\n")
+            references = [Reference(output=Output(text=row["Answer"]), tags=[CORRECT_TAG])]
+            instance = Instance(
+                id=f"id{row_index}",
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/audio_language/air_bench_chat_scenario.py

@@ -93,7 +93,8 @@ class AirBenchChatScenario(Scenario):
         meta_data = json.load(open(meta_data_path))
         subject_indices = self._get_subject_indices(meta_data)
         valid_testing_indices = []
-        for _, row in enumerate(subject_indices):
+
+        for _, row in tqdm(enumerate(subject_indices)):
             audio_meda_data = meta_data[row]
             hf_audio_file_path = os.path.join(
                 self.HF_DATA_PATH_PREFIX,
@@ -105,6 +106,7 @@ class AirBenchChatScenario(Scenario):
             ensure_file_downloaded(source_url=hf_audio_file_path, target_path=local_audio_file_path)
             if not is_invalid_audio_file(local_audio_file_path):
                 valid_testing_indices.append(row)
+
         for _, row in enumerate(tqdm(valid_testing_indices)):
             audio_meda_data_valid = meta_data[row]
             local_audio_file_path_valid = os.path.join(

helm/benchmark/scenarios/audio_language/air_bench_foundation_scenario.py

@@ -53,9 +53,9 @@ class AirBenchFoundationScenario(Scenario):
     META_DATA_FILE_PATH = (
         "https://huggingface.co/datasets/qyang1021/AIR-Bench-Dataset/resolve/main/Foundation/Foundation_meta.json"
     )
-    SUJECTS_DICT = {
+    SUBJECTS_DICT = {
         "music_instrument_classification": "Music_Instruments_Classfication",
-        "music_genera_recognition": "Music_Genre_Recognition",
+        "music_genre_recognition": "Music_Genre_Recognition",
         "music_qa": "Music_AQA",
     }
     OPTION_KEYS = ["choice_a", "choice_b", "choice_c", "choice_d"]
@@ -68,15 +68,15 @@ class AirBenchFoundationScenario(Scenario):
     def __init__(self, subject: str) -> None:
         super().__init__()
 
-        if subject not in AirBenchFoundationScenario.SUJECTS_DICT.keys():
-            raise ValueError(f"Invalid subject. Valid subjects are: {AirBenchFoundationScenario.SUJECTS_DICT.keys()}")
+        if subject not in AirBenchFoundationScenario.SUBJECTS_DICT.keys():
+            raise ValueError(f"Invalid subject. Valid subjects are: {AirBenchFoundationScenario.SUBJECTS_DICT.keys()}")
 
         self._subject: str = subject
 
     def _get_subject_indices(self, meta_data) -> List[int]:
         subject_indices = []
         for idx, line in enumerate(meta_data):
-            if line["task_name"] == self.SUJECTS_DICT[self._subject]:
+            if line["task_name"] == self.SUBJECTS_DICT[self._subject]:
                 subject_indices.append(idx)
         return subject_indices
 

helm/benchmark/scenarios/audio_language/casual_conversations2_scenario.py

@@ -19,7 +19,7 @@ from helm.common.audio_utils import extract_audio
 
 
 class CasualConversations2Scenario(Scenario):
-    """
+    r"""
     Casual Conversation v2 (Porgali et al, 2023) is composed of over 5,567 participants (26,467 videos).
     The videos feature paid individuals who agreed to participate in the project and explicitly provided
     Age, Gender, Language/Dialect, Geo-location, Disability, Physical adornments, Physical attributes labels