crfm-helm 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

helm/benchmark/run_specs/speech_disorder_audio_run_specs.py

@@ -112,9 +112,13 @@ def get_ultra_suite_asr_classification_run_spec() -> RunSpec:
     )
     adapter_spec = _get_generation_adapter_spec(
         instructions="""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording is provided to you, typically consisting of a speech prompt from a pathologist followed by a child's repetition. Based on your expertise transcribe the child's speech into text. Do not make any assumptions about the words the child is expected to say. Only transcribe based on the words that the child actually says. Only respond with the text transcription, no other text or commentary.""",  # noqa: E501
-        max_tokens=10,
+        max_tokens=50,
     )
-    metric_specs: List[MetricSpec] = audio_classification_metric_specs()
+    metric_specs: List[MetricSpec] = [
+        MetricSpec(
+            class_name="helm.benchmark.metrics.ultra_suite_asr_classification_metrics.UltraSuiteASRMetric", args={}
+        )
+    ]
     run_spec_name: str = "ultra_suite_asr_classification"
     return RunSpec(
         name=run_spec_name,

helm/benchmark/scenarios/aci_bench_scenario.py

@@ -1,6 +1,7 @@
 import json
 import os
 from typing import List
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -10,6 +11,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_file_downloaded
 
@@ -124,3 +126,24 @@ class ACIBenchScenario(Scenario):
             instances.extend(self.process_json(test_json, TEST_SPLIT))
 
         return instances
+
+    def get_metadata(self):
+        return ScenarioMetadata(
+            name="aci_bench",
+            display_name="ACI-Bench",
+            description="ACI-Bench is a benchmark of real-world patient-doctor conversations paired "
+            "with structured clinical notes. The benchmark evaluates a model's ability to "
+            "understand spoken medical dialogue and convert it into formal clinical "
+            "documentation, covering sections such as history of present illness, physical "
+            "exam findings, results, and assessment and plan [(Yim et al., "
+            "2024)](https://www.nature.com/articles/s41597-023-02487-3).",
+            taxonomy=TaxonomyInfo(
+                task="Text generation",
+                what="Extract and structure information from patient-doctor " "conversations",
+                when="Any",
+                who="Clinician",
+                language="English",
+            ),
+            main_metric="aci_bench_accuracy",
+            main_split="test",
+        )

helm/benchmark/scenarios/air_bench_scenario.py

@@ -2,6 +2,7 @@ import datasets
 import os
 from typing import List
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.benchmark.scenarios.scenario import (
     Scenario,
     Instance,
@@ -9,6 +10,7 @@ from helm.benchmark.scenarios.scenario import (
     TEST_SPLIT,
     Input,
     Output,
+    ScenarioMetadata,
 )
 from helm.common.general import ensure_directory_exists
 
@@ -53,3 +55,22 @@ class AIRBench2024Scenario(Scenario):
             instance = Instance(input=input, references=references, split=TEST_SPLIT)
             instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="air_bench_2024",
+            display_name="AIRBench 2024",
+            description="AIRBench 2024 is a AI safety benchmark that aligns with emerging government "
+            "regulations and company policies. It consists of diverse, malicious prompts "
+            "spanning categories of the regulation-based safety categories in the AIR 2024 "
+            "safety taxonomy.\n",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction-following text generation",
+                what="malicious prompts",
+                when="2024",
+                who="dataset authors and language models",
+                language="English",
+            ),
+            main_metric="air_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/alrage_scenario.py

@@ -0,0 +1,54 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+
+
+class ALRAGEScenario(Scenario):
+    """ALRAGE"""  # noqa: E501
+
+    name = "alrage"
+    description = "ALRAGE"
+    tags = ["open-book question answering"]
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset: datasets.Dataset = datasets.load_dataset(
+            "OALL/ALRAGE",
+            revision="4827b2ed2436aea578e84d9bd4150b66ab8bbe0e",
+            split="train",
+            cache_dir=cache_dir,
+        )
+
+        # Read all instances
+        instances: List[Instance] = []
+        for row in dataset:
+            input = Input(text=f"السؤال:\n{row['question']}\n\nالسياقات المقترحة:\n{row['candidates']}\n")
+            references: List[Reference] = []
+            references = [
+                Reference(
+                    output=Output(text=row["gold_answer"]),
+                    tags=[CORRECT_TAG],
+                )
+            ]
+            instance = Instance(
+                id=row["id"],
+                input=input,
+                references=references,
+                split=TEST_SPLIT,
+            )
+            instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/anthropic_hh_rlhf_scenario.py

@@ -2,7 +2,8 @@ import re
 from typing import List, Any, Dict
 from datasets import load_dataset
 
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TRAIN_SPLIT, TEST_SPLIT
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TRAIN_SPLIT, TEST_SPLIT, ScenarioMetadata
 
 
 class AnthropicHHRLHFScenario(Scenario):
@@ -88,3 +89,24 @@ class AnthropicHHRLHFScenario(Scenario):
                 )
                 instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="anthropic_hh_rlhf",
+            display_name="Anthropic RLHF dataset",
+            short_display_name="Anthropic RLHF dataset",
+            description="The dialogue datasets released by Anthropic to facilitate research in model "
+            "helpfulness and harmlessness ([Bai et al., "
+            "2022](https://arxiv.org/pdf/2204.05862.pdf); [Ganguli et al., "
+            "2022](https://arxiv.org/pdf/2209.07858.pdf)). We only use the first utterance "
+            "of each dialogue.",
+            taxonomy=TaxonomyInfo(
+                task="open-ended instruction following",
+                what="Human-LM dialogues and preference labels",
+                when="2022",
+                who="Workers from MTurk and Upwork, language models from Anthropic",
+                language="English",
+            ),
+            main_metric="Helpfulness",
+            main_split="test",
+        )

helm/benchmark/scenarios/anthropic_red_team_scenario.py

@@ -2,7 +2,8 @@ import re
 from typing import List, Any, Dict
 from datasets import load_dataset
 
-from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TRAIN_SPLIT, TEST_SPLIT
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
+from helm.benchmark.scenarios.scenario import Scenario, Instance, Input, TRAIN_SPLIT, TEST_SPLIT, ScenarioMetadata
 
 
 class AnthropicRedTeamScenario(Scenario):
@@ -69,3 +70,13 @@ class AnthropicRedTeamScenario(Scenario):
                 )
                 instances.append(instance)
         return instances
+
+    def get_metadata(self) -> ScenarioMetadata:
+        return ScenarioMetadata(
+            name="anthropic_red_team",
+            display_name="Anthropic Red Team",
+            description="Anthropic Red Team",
+            taxonomy=TaxonomyInfo(task="instruction following sfaety", what="?", when="?", who="?", language="English"),
+            main_metric="safety_score",
+            main_split="test",
+        )

helm/benchmark/scenarios/arabic_exams_scenario.py

@@ -0,0 +1,114 @@
+import os
+from typing import List
+
+import datasets
+
+from helm.common.general import ensure_directory_exists
+from helm.benchmark.scenarios.scenario import (
+    Scenario,
+    Instance,
+    Reference,
+    TEST_SPLIT,
+    TRAIN_SPLIT,
+    CORRECT_TAG,
+    Input,
+    Output,
+)
+from helm.common.hierarchical_logger import hwarn
+
+
+class ArabicEXAMSScenario(Scenario):
+    """The Arabic subset of the EXAMS High School Examinations Dataset for Multilingual Question Answering
+
+    We use the Open Arabic LLM Leaderboard (OALL) version mirror of the Arabic subset of EXAMS, which is in-turn based
+    on the AceGPT version.
+
+    See: https://www.tii.ae/news/introducing-open-arabic-llm-leaderboard-empowering-arabic-language-modeling-community
+
+    References:
+
+    ```
+    @misc{huang2024acegptlocalizinglargelanguage,
+        title={AceGPT, Localizing Large Language Models in Arabic},
+        author={Huang Huang and Fei Yu and Jianqing Zhu and Xuening Sun and Hao Cheng and Dingjie Song and Zhihong Chen and Abdulmohsen Alharthi and Bang An and Juncai He and Ziche Liu and Zhiyi Zhang and Junying Chen and Jianquan Li and Benyou Wang and Lian Zhang and Ruoyu Sun and Xiang Wan and Haizhou Li and Jinchao Xu},
+        year={2024},
+        eprint={2309.12053},
+        archivePrefix={arXiv},
+        primaryClass={cs.CL},
+        url={https://arxiv.org/abs/2309.12053},
+    }```
+
+    ```
+    @inproceedings{hardalov-etal-2020-exams,
+        title = "{EXAMS}: A Multi-subject High School Examinations Dataset for Cross-lingual and Multilingual Question Answering",
+        author = "Hardalov, Momchil  and
+        Mihaylov, Todor  and
+        Zlatkova, Dimitrina  and
+        Dinkov, Yoan  and
+        Koychev, Ivan  and
+        Nakov, Preslav",
+        editor = "Webber, Bonnie  and
+        Cohn, Trevor  and
+        He, Yulan  and
+        Liu, Yang",
+        booktitle = "Proceedings of the 2020 Conference on Empirical Methods in Natural Language Processing (EMNLP)",
+        month = nov,
+        year = "2020",
+        address = "Online",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2020.emnlp-main.438/",
+        doi = "10.18653/v1/2020.emnlp-main.438",
+        pages = "5427--5444",
+        abstract = "We propose EXAMS {--} a new benchmark dataset for cross-lingual and multilingual question answering for high school examinations. We collected more than 24,000 high-quality high school exam questions in 16 languages, covering 8 language families and 24 school subjects from Natural Sciences and Social Sciences, among others.EXAMS offers unique fine-grained evaluation framework across multiple languages and subjects, which allows precise analysis and comparison of the proposed models. We perform various experiments with existing top-performing multilingual pre-trained models and show that EXAMS offers multiple challenges that require multilingual knowledge and reasoning in multiple domains. We hope that EXAMS will enable researchers to explore challenging reasoning and knowledge transfer methods and pre-trained models for school question answering in various languages which was not possible by now. The data, code, pre-trained models, and evaluation are available at http://github.com/mhardalov/exams-qa."
+    }```
+    """  # noqa: E501
+
+    name = "arabic_exams"
+    description = "EXAMS is a benchmark dataset for multilingual and cross-lingual question answering from high school examinations. "  # noqa: E501
+    tags = ["knowledge", "multiple_choice"]
+
+    CHOICES = ["A", "B", "C", "D"]
+    # Remap validation split to train split
+    HF_SPLIT_TO_HELM_SPLIT = {"validation": TRAIN_SPLIT, "test": TEST_SPLIT}
+
+    def __init__(self, subject: str):
+        super().__init__()
+        self.subject: str = subject.replace("_", " ")
+
+    def get_instances(self, output_path: str) -> List[Instance]:
+        cache_dir = os.path.join(output_path, "data")
+        ensure_directory_exists(cache_dir)
+        dataset_splits = datasets.load_dataset(
+            "OALL/Arabic_EXAMS",
+            revision="bc7a29346dbcaa16a8cd883b1f3e681ab2b7ff2a",
+            cache_dir=cache_dir,
+        )
+
+        instances: List[Instance] = []
+        for split_name, dataset in dataset_splits.items():
+            for row in dataset:
+                subject = row["id"].split("-")[0]
+                if self.subject != "all" and self.subject != subject:
+                    continue
+                input = Input(text=row["question"])
+                references: List[Reference] = []
+                if row["answer"] not in self.CHOICES:
+                    hwarn(f"Invalid value in answer column in row: {row}")
+                    continue
+                correct_choice = row["answer"]
+                for choice in self.CHOICES:
+                    references.append(
+                        Reference(
+                            output=Output(text=row[choice]),
+                            tags=[CORRECT_TAG] if choice == correct_choice else [],
+                        )
+                    )
+                instance = Instance(
+                    id=row["id"],
+                    input=input,
+                    references=references,
+                    split=self.HF_SPLIT_TO_HELM_SPLIT[split_name],
+                )
+                instances.append(instance)
+
+        return instances

helm/benchmark/scenarios/arabic_mmlu_scenario.py

@@ -19,8 +19,6 @@ from helm.benchmark.scenarios.scenario import (
 class ArabicMMLUScenario(Scenario):
     """ArabicMMLU
 
-    EXPERIMENTAL: This scenario may have future reverse incompatible changes.
-
     ArabicMMLU is the first multi-task language understanding benchmark
     for Arabic language, sourced from school exams across diverse educational
     levels in different countries spanning North Africa, the Levant, and the
@@ -39,12 +37,16 @@ class ArabicMMLUScenario(Scenario):
     OPTIONS = ["A", "B", "C", "D"]
     HF_SPLIT_TO_HELM_SPLIT = {"dev": TRAIN_SPLIT, "test": TEST_SPLIT}
 
+    def __init__(self, subset: str):
+        super().__init__()
+        self.subset = subset.replace("_", " ")
+
     def get_instances(self, output_path: str) -> List[Instance]:
         cache_dir = os.path.join(output_path, "data")
         ensure_directory_exists(cache_dir)
         dataset_splits: Dict[str, datasets.Dataset] = datasets.load_dataset(
             "MBZUAI/ArabicMMLU",
-            "All",
+            self.subset,
             revision="7aa530e2893ac420352b3f5c1a1310c010e9758b",
             cache_dir=cache_dir,
         )
@@ -63,7 +65,9 @@ class ArabicMMLUScenario(Scenario):
                         continue
                     references.append(
                         Reference(
-                            output=Output(text=row[column_name]),
+                            # Need to convert column to string because the references are floats
+                            # for the subject "Math (Primary School)"
+                            output=Output(text=str(row[column_name])),
                             tags=[CORRECT_TAG] if option_index == correct_option_index else [],
                         )
                     )

helm/benchmark/scenarios/aratrust_scenario.py

@@ -47,8 +47,25 @@ class AraTrustScenario(Scenario):
     description = "aratrust"
     tags = ["trustworthiness"]
 
+    CATEGORIES = [
+        "Ethics",
+        "Illegal",
+        "Mental Health",
+        "Offensive",
+        "Physical Health",
+        "Privacy",
+        "Trustfulness",
+        "Unfairness",
+    ]
     OPTION_KEYS = ["A", "B", "C"]
 
+    def __init__(self, category: str):
+        super().__init__()
+        category = category.replace("_", " ")
+        if category not in self.CATEGORIES and category != "all":
+            raise Exception(f"Unknown category {category}")
+        self.category = category
+
     def get_instances(self, output_path: str) -> List[Instance]:
         cache_dir = os.path.join(output_path, "data")
         ensure_directory_exists(cache_dir)
@@ -60,6 +77,8 @@ class AraTrustScenario(Scenario):
         )
         instances: List[Instance] = []
         for row_index, row in enumerate(dataset):
+            if self.category != "all" and self.category != row["Category"]:
+                continue
             question_text = row["Question"]
             option_texts = [row[option_key] for option_key in self.OPTION_KEYS if row[option_key]]
             joined_option_texts = "\n".join(option_texts)

helm/benchmark/scenarios/audio_language/ultra_suite_asr_classification_scenario.py

@@ -1,7 +1,7 @@
-from typing import List, Tuple
+from typing import List
 import os
-import json
 
+from datasets import load_dataset
 from tqdm import tqdm
 
 from helm.benchmark.scenarios.scenario import (
@@ -14,38 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from huggingface_hub import snapshot_download
-
-
-def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
-    """
-    Find all pairs of MP3 and JSON files in the given directory and its subdirectories.
-    Each pair consists of an MP3 file and its corresponding JSON file with the same base name.
-
-    Args:
-        directory: Path to the directory containing the files
-
-    Returns:
-        List of tuples where each tuple contains (mp3_path, json_path)
-    """
-    pairs = []
-
-    # Walk through all directories and subdirectories
-    for root, _, files in os.walk(directory):
-        # Get all MP3 files in current directory
-        mp3_files = [f for f in files if f.endswith(".mp3")]
-
-        for mp3_file in mp3_files:
-            base_name = os.path.splitext(mp3_file)[0]
-            json_file = f"{base_name}.json"
-
-            # Check if corresponding JSON file exists in the same directory
-            if json_file in files:
-                mp3_path = os.path.join(root, mp3_file)
-                json_path = os.path.join(root, json_file)
-                pairs.append((mp3_path, json_path))
-
-    return pairs
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
 
 
 class UltraSuiteASRClassificationScenario(Scenario):
@@ -59,9 +28,6 @@ class UltraSuiteASRClassificationScenario(Scenario):
     description = "A scenario for evaluating speech disorders in children"
     tags = ["audio", "classification", "speech_disorder", "asr"]
 
-    # Classification options
-    options: List[str] = ["Healthy", "Unhealthy"]
-
     def get_instances(self, output_path: str) -> List[Instance]:
         """
         Create instances from the audio files and their corresponding JSON annotations.
@@ -69,36 +35,40 @@ class UltraSuiteASRClassificationScenario(Scenario):
         - Audio files (e.g., .mp3)
         - A JSON file with annotations containing 'answer' field
         """
-        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
-        data_path = snapshot_download(
-            repo_id="SAA-Lab/SLPHelmManualLabels",
-            repo_type="dataset",
-            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
-        )
+
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        os.makedirs(audio_save_dir, exist_ok=True)
+
+        print("Downloading SAA-Lab/SLPHelmUltraSuitePlus dataset...")
+        dataset = load_dataset("SAA-Lab/SLPHelmUltraSuitePlus")
 
         instances: List[Instance] = []
         split: str = TEST_SPLIT
 
-        # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(data_path)
+        for idx, row in enumerate(tqdm(dataset["train"])):
 
-        for audio_path, json_path in tqdm(pairs):
+            label = row["disorder_class"]
+            transcription = row["transcription"]
 
-            # Load the annotation
-            with open(json_path, "r") as f:
-                annotation = json.load(f)
+            unique_id = str(idx)
+            local_audio_name = f"{label}_{unique_id}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
 
-            # Get the correct answer and convert to label
-            answer = annotation["disorder_class"]
             # Create references for each option
-            references: List[Reference] = [Reference(Output(text=answer), tags=[CORRECT_TAG])]
+            references: List[Reference] = []
+            for option in ["typically_developing", "speech_disorder"]:
+                reference = Reference(Output(text=option), tags=[CORRECT_TAG] if option == label else [])
+                references.append(reference)
 
             # Create the input with audio and instruction
             content = [
-                MediaObject(content_type="audio/mpeg", location=audio_path),
+                MediaObject(content_type="audio/mpeg", location=local_audio_path),
             ]
 
             input = Input(multimedia_content=MultimediaObject(content))
-            instances.append(Instance(input=input, references=references, split=split))
+            instances.append(
+                Instance(input=input, references=references, split=split, extra_data={"transcription": transcription})
+            )
 
         return instances

helm/benchmark/scenarios/audio_language/ultra_suite_asr_transcription_scenario.py

@@ -1,7 +1,7 @@
-from typing import List, Tuple
+from typing import List
 import os
-import json
 
+from datasets import load_dataset
 from tqdm import tqdm
 
 from helm.benchmark.scenarios.scenario import (
@@ -14,38 +14,7 @@ from helm.benchmark.scenarios.scenario import (
     Output,
 )
 from helm.common.media_object import MediaObject, MultimediaObject
-from huggingface_hub import snapshot_download
-
-
-def find_audio_json_pairs(directory: str) -> List[Tuple[str, str]]:
-    """
-    Find all pairs of MP3 and JSON files in the given directory and its subdirectories.
-    Each pair consists of an MP3 file and its corresponding JSON file with the same base name.
-
-    Args:
-        directory: Path to the directory containing the files
-
-    Returns:
-        List of tuples where each tuple contains (mp3_path, json_path)
-    """
-    pairs = []
-
-    # Walk through all directories and subdirectories
-    for root, _, files in os.walk(directory):
-        # Get all MP3 files in current directory
-        mp3_files = [f for f in files if f.endswith(".mp3")]
-
-        for mp3_file in mp3_files:
-            base_name = os.path.splitext(mp3_file)[0]
-            json_file = f"{base_name}.json"
-
-            # Check if corresponding JSON file exists in the same directory
-            if json_file in files:
-                mp3_path = os.path.join(root, mp3_file)
-                json_path = os.path.join(root, json_file)
-                pairs.append((mp3_path, json_path))
-
-    return pairs
+from helm.common.audio_utils import ensure_audio_file_exists_from_array
 
 
 class UltraSuiteASRTranscriptionScenario(Scenario):
@@ -66,31 +35,33 @@ class UltraSuiteASRTranscriptionScenario(Scenario):
         - Audio files (e.g., .mp3)
         - A JSON file with annotations containing 'answer' field
         """
-        print("Downloading SAA-Lab/SLPHelmManualLabels dataset...")
-        data_path = snapshot_download(
-            repo_id="SAA-Lab/SLPHelmManualLabels",
-            repo_type="dataset",
-            revision="38c2d7dab831acf8ccff0ca6f6463d6a8a0184ed",
-        )
+        audio_save_dir = os.path.join(output_path, "audio_files")
+        os.makedirs(audio_save_dir, exist_ok=True)
+
+        print("Downloading SAA-Lab/SLPHelmUltraSuitePlus dataset...")
+        dataset = load_dataset("SAA-Lab/SLPHelmUltraSuitePlus")
 
         instances: List[Instance] = []
         split: str = TEST_SPLIT
 
         # Find all pairs of audio and JSON files
-        pairs = find_audio_json_pairs(data_path)
-
-        for audio_path, json_path in tqdm(pairs):
+        for idx, row in enumerate(tqdm(dataset["train"])):
 
             # Load the annotation
-            with open(json_path, "r") as f:
-                annotation = json.load(f)
+            # Load the annotation
+            label = row["disorder_class"]
+
+            unique_id = str(idx)
+            local_audio_name = f"{label}_{unique_id}.mp3"
+            local_audio_path = os.path.join(audio_save_dir, local_audio_name)
+            ensure_audio_file_exists_from_array(local_audio_path, row["audio"]["array"], row["audio"]["sampling_rate"])
 
-            # Create references for the transcription
-            references: List[Reference] = [Reference(Output(text=annotation["transcription"]), tags=[CORRECT_TAG])]
+            # Create references for each option
+            references: List[Reference] = [Reference(Output(text=row["transcription"]), tags=[CORRECT_TAG])]
 
             # Create the input with audio and instruction
             content = [
-                MediaObject(content_type="audio/mpeg", location=audio_path),
+                MediaObject(content_type="audio/mpeg", location=local_audio_path),
             ]
 
             input = Input(multimedia_content=MultimediaObject(content))