crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/run_specs/mmlu_clinical_afr_run_specs.py

@@ -0,0 +1,49 @@
+"""Run spec functions for three clinical sections of MMLU human-translated into 11 African languages
+
+Available subjects: "clinical_knowledge", "college_medicine", "virology"
+Available langs: "af", "zu", "xh", "am", "bm", "ig", "nso", "sn", "st", "tn", "ts" (see lang_map below for language code mapping to language name, or here for ISO code reference: https://huggingface.co/languages)
+"""  # noqa: E501
+
+from helm.benchmark.adaptation.adapter_spec import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("mmlu_clinical_afr")
+def get_mmlu_clinical_afr_spec(subject: str, lang: str, method: str = ADAPT_MULTIPLE_CHOICE_JOINT) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.mmlu_clinical_afr_scenario.MMLU_Clinical_Afr_Scenario",
+        args={"subject": subject, "lang": lang},
+    )
+
+    lang_map = {
+        "af": "Afrikaans",
+        "zu": "Zulu",
+        "xh": "Xhosa",
+        "am": "Amharic",
+        "bm": "Bambara",
+        "ig": "Igbo",
+        "nso": "Sepedi",
+        "sn": "Shona",
+        "st": "Sesotho",
+        "tn": "Setswana",
+        "ts": "Tsonga",
+    }
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=method,
+        instructions=f"The following are multiple choice questions (with answers) about {subject.replace('_', ' ')} "
+        f"in {lang_map[lang]}.",
+        input_noun="Question",
+        output_noun="Answer",
+    )
+
+    return RunSpec(
+        name=f"mmlu_clinical_afr:subject={subject},lang={lang},method={method}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=[f"mmlu_clinical_afr_{subject}", f"mmlu_clinical_afr_{subject}_{lang}"],
+    )

helm/benchmark/run_specs/oab_exams_specs.py

@@ -0,0 +1,32 @@
+from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("oab_exams")
+def get_enem_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.oab_exams_scenario.OABExamsScenario", args={})
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="Dê uma resposta selecionando uma letra entre as opções fornecidas. "
+        "Se as opções forem A, B, C e D,"
+        "sua resposta deve consistir em uma única letra que corresponde a resposta correta.\n"
+        "Exemplo: Ao conselho da subseção compete\nA. representar a OAB no Conselho de Segurança do MERCOSUL."
+        "\nB. fiscalizar as funções e atribuições do conselho seccional.\nC. instaurar e instruir processos "
+        "disciplinares, para julgamento pelo Conselho Federal.\nD. receber pedido de inscrição nos quadros de "
+        "advogado e estagiário, instruindo e emitindo parecer prévio, para decisão do conselho seccional.\n"
+        "Resposta: D",
+        input_noun="Pergunta",
+        output_noun="Resposta",
+    )
+
+    return RunSpec(
+        name="oab_exams",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["oab_exams"],
+    )

helm/benchmark/run_specs/safety_run_specs.py

@@ -41,6 +41,43 @@ def get_harm_bench_spec() -> RunSpec:
     )
 
 
+@run_spec_function("harm_bench_gcg_transfer")
+def get_harm_bench_gcg_transfer_spec() -> RunSpec:
+    adapter_spec = AdapterSpec(
+        method=ADAPT_GENERATION,
+        global_prefix="",
+        global_suffix="",
+        instructions="",
+        input_prefix="",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        instance_prefix="",
+        max_train_instances=0,
+        num_outputs=1,
+        max_tokens=512,
+        temperature=0.0,
+        stop_sequences=[],
+    )
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.harm_bench_gcg_transfer_scenario.HarmBenchGCGTransferScenario"
+    )
+    annotator_specs = [AnnotatorSpec(class_name="helm.benchmark.annotation.harm_bench_annotator.HarmBenchAnnotator")]
+    metric_specs = [
+        MetricSpec(class_name="helm.benchmark.metrics.safety_metrics.SafetyScoreMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.safety_metrics.SafetyBasicGenerationMetric"),
+        MetricSpec(class_name="helm.benchmark.metrics.basic_metrics.InstancesPerSplitMetric"),
+    ]
+    return RunSpec(
+        name="harm_bench_gcg_transfer",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        annotators=annotator_specs,
+        groups=["harm_bench_gcg_transfer"],
+    )
+
+
 @run_spec_function("simple_safety_tests")
 def get_simple_safety_tests_spec() -> RunSpec:
     adapter_spec = AdapterSpec(

helm/benchmark/run_specs/{bhasa_run_specs.py → seahelm_run_specs.py}

@@ -5,9 +5,9 @@ from helm.benchmark.adaptation.common_adapter_specs import (
     get_generation_adapter_spec,
     get_multiple_choice_separate_adapter_spec,
 )
-from helm.benchmark.metrics.bhasa_metrics_specs import (
-    get_bhasa_machine_translation_metric_specs,
-    get_bhasa_qa_metric_specs,
+from helm.benchmark.metrics.seahelm_metrics_specs import (
+    get_seahelm_machine_translation_metric_specs,
+    get_seahelm_qa_metric_specs,
 )
 from helm.benchmark.metrics.common_metric_specs import (
     get_basic_metric_specs,
@@ -17,7 +17,7 @@ from helm.benchmark.metrics.common_metric_specs import (
 from helm.benchmark.run_spec import RunSpec, run_spec_function
 from helm.benchmark.scenarios.scenario import ScenarioSpec
 
-# BHASA Run Specs
+# SEA-HELM Run Specs
 #   A. Natural Language Understanding
 #   B. Natural Language Generation
 #   C. Natural Language Reasoning
@@ -43,18 +43,18 @@ def get_tydiqa_spec() -> RunSpec:
         max_tokens=256,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.TyDiQAScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.TyDiQAScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
-        metric_specs=get_bhasa_qa_metric_specs(
+        metric_specs=get_seahelm_qa_metric_specs(
             args={
                 "language": "id",
             }
         ),
-        groups=["bhasa_nlu", "tydiqa"],
+        groups=["seahelm_nlu", "tydiqa"],
     )
 
 
@@ -84,7 +84,7 @@ def get_xquad_spec(language="th") -> RunSpec:
     )
 
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.bhasa_scenario.XQuADScenario",
+        class_name="helm.benchmark.scenarios.seahelm_scenario.XQuADScenario",
         args={
             "language": language,
         },
@@ -94,12 +94,12 @@ def get_xquad_spec(language="th") -> RunSpec:
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
-        metric_specs=get_bhasa_qa_metric_specs(
+        metric_specs=get_seahelm_qa_metric_specs(
             args={
                 "language": language,
             }
         ),
-        groups=["bhasa_nlu", f"xquad_{language}"],
+        groups=["seahelm_nlu", f"xquad_{language}"],
     )
 
 
@@ -116,18 +116,18 @@ def get_indicqa_spec() -> RunSpec:
         max_tokens=256,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.IndicQAScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.IndicQAScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
-        metric_specs=get_bhasa_qa_metric_specs(
+        metric_specs=get_seahelm_qa_metric_specs(
             args={
                 "language": "ta",
             }
         ),
-        groups=["bhasa_nlu", "indicqa"],
+        groups=["seahelm_nlu", "indicqa"],
     )
 
 
@@ -146,14 +146,14 @@ def get_nusax_spec() -> RunSpec:
         max_tokens=16,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.NusaXScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.NusaXScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() + get_classification_metric_specs(),
-        groups=["bhasa_nlu", "nusax"],
+        groups=["seahelm_nlu", "nusax"],
     )
 
 
@@ -171,14 +171,14 @@ def get_uitvsfc_spec() -> RunSpec:
         max_tokens=16,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.UITVSFCScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.UITVSFCScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() + get_classification_metric_specs(),
-        groups=["bhasa_nlu", "uitvsfc"],
+        groups=["seahelm_nlu", "uitvsfc"],
     )
 
 
@@ -196,14 +196,14 @@ def get_wisesight_spec() -> RunSpec:
         max_tokens=16,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.WisesightScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.WisesightScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() + get_classification_metric_specs(),
-        groups=["bhasa_nlu", "wisesight"],
+        groups=["seahelm_nlu", "wisesight"],
     )
 
 
@@ -221,14 +221,14 @@ def get_indicsentiment_spec() -> RunSpec:
         max_tokens=16,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.IndicSentimentScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.IndicSentimentScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_classification_metric_specs() + get_basic_metric_specs([]),
-        groups=["bhasa_nlu", "indicsentiment"],
+        groups=["seahelm_nlu", "indicsentiment"],
     )
 
 
@@ -250,14 +250,14 @@ def get_mlhsd_spec() -> RunSpec:
         max_tokens=16,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.MLHSDScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.MLHSDScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() + get_classification_metric_specs(),
-        groups=["bhasa_nlu", "mlhsd"],
+        groups=["seahelm_nlu", "mlhsd"],
     )
 
 
@@ -278,14 +278,14 @@ def get_vihsd_spec() -> RunSpec:
         max_tokens=16,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.ViHSDScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.ViHSDScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() + get_classification_metric_specs(),
-        groups=["bhasa_nlu", "vihsd"],
+        groups=["seahelm_nlu", "vihsd"],
     )
 
 
@@ -304,14 +304,14 @@ def get_thaitoxicitytweets_spec() -> RunSpec:
         max_tokens=16,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.ThaiToxicityTweetsScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.ThaiToxicityTweetsScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() + get_classification_metric_specs(),
-        groups=["bhasa_nlu", "thaitoxicitytweets"],
+        groups=["seahelm_nlu", "thaitoxicitytweets"],
     )
 
 
@@ -378,7 +378,7 @@ def get_flores_spec(source="en", target="id") -> RunSpec:
     )
 
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.bhasa_scenario.FloresScenario",
+        class_name="helm.benchmark.scenarios.seahelm_scenario.FloresScenario",
         args={
             "pair": pair,
         },
@@ -388,8 +388,8 @@ def get_flores_spec(source="en", target="id") -> RunSpec:
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
-        metric_specs=get_bhasa_machine_translation_metric_specs(),
-        groups=["bhasa_nlg", f"flores_{pair}"],
+        metric_specs=get_seahelm_machine_translation_metric_specs(),
+        groups=["seahelm_nlg", f"flores_{pair}"],
     )
 
 
@@ -414,14 +414,14 @@ def get_indonli_spec() -> RunSpec:
         max_tokens=2,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.IndoNLIScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.IndoNLIScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() + get_classification_metric_specs(),
-        groups=["bhasa_nlr", "indonli"],
+        groups=["seahelm_nlr", "indonli"],
     )
 
 
@@ -456,7 +456,7 @@ def get_xnli_spec(language="vi") -> RunSpec:
     )
 
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.bhasa_scenario.XNLIScenario",
+        class_name="helm.benchmark.scenarios.seahelm_scenario.XNLIScenario",
         args={
             "language": language,
         },
@@ -467,7 +467,7 @@ def get_xnli_spec(language="vi") -> RunSpec:
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() + get_classification_metric_specs(),
-        groups=["bhasa_nlr", f"xnli_{language}"],
+        groups=["seahelm_nlr", f"xnli_{language}"],
     )
 
 
@@ -487,14 +487,14 @@ def get_indicxnli_spec() -> RunSpec:
         max_tokens=2,
     )
 
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bhasa_scenario.IndicXNLIScenario")
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.seahelm_scenario.IndicXNLIScenario")
 
     return RunSpec(
         name=name,
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() + get_classification_metric_specs(),
-        groups=["bhasa_nlr", "indicxnli"],
+        groups=["seahelm_nlr", "indicxnli"],
     )
 
 
@@ -531,7 +531,7 @@ def get_xcopa_spec(language="id") -> RunSpec:
     )
 
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.bhasa_scenario.XCOPAScenario",
+        class_name="helm.benchmark.scenarios.seahelm_scenario.XCOPAScenario",
         args={
             "language": language,
         },
@@ -542,7 +542,7 @@ def get_xcopa_spec(language="id") -> RunSpec:
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs() + get_classification_metric_specs(),
-        groups=["bhasa_nlr", f"xcopa_{language}"],
+        groups=["seahelm_nlr", f"xcopa_{language}"],
     )
 
 
@@ -566,7 +566,7 @@ def get_lindsea_syntax_minimal_pairs_spec(language: str = "id", method: str = "m
         )
 
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.bhasa_scenario.LINDSEASyntaxMinimalPairsScenario",
+        class_name="helm.benchmark.scenarios.seahelm_scenario.LINDSEASyntaxMinimalPairsScenario",
         args={
             "method": method,
             "language": language,
@@ -579,7 +579,7 @@ def get_lindsea_syntax_minimal_pairs_spec(language: str = "id", method: str = "m
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs(),
         groups=[
-            "bhasa_linguistic",
+            "seahelm_linguistic",
             f"lindsea_syntax_minimal_pairs_{language}",
             f"lindsea_syntax_minimal_pairs_{method}_{language}",
         ],
@@ -599,7 +599,7 @@ def get_lindsea_pragmatics_presuppositions_spec(language: str = "id", subset: st
     )
 
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.bhasa_scenario.LINDSEAPragmaticsPresuppositionsScenario",
+        class_name="helm.benchmark.scenarios.seahelm_scenario.LINDSEAPragmaticsPresuppositionsScenario",
         args={
             "language": language,
             "subset": subset,
@@ -612,7 +612,7 @@ def get_lindsea_pragmatics_presuppositions_spec(language: str = "id", subset: st
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs(),
         groups=[
-            "bhasa_linguistic",
+            "seahelm_linguistic",
             f"lindsea_pragmatics_presuppositions_{language}",
             f"lindsea_pragmatics_presuppositions_{subset}_{language}",
         ],
@@ -632,7 +632,7 @@ def get_lindsea_pragmatics_scalar_implicatures_spec(language: str = "id", subset
     )
 
     scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.bhasa_scenario.LINDSEAPragmaticsScalarImplicaturesScenario",
+        class_name="helm.benchmark.scenarios.seahelm_scenario.LINDSEAPragmaticsScalarImplicaturesScenario",
         args={
             "language": language,
             "subset": subset,
@@ -645,7 +645,7 @@ def get_lindsea_pragmatics_scalar_implicatures_spec(language: str = "id", subset
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs(),
         groups=[
-            "bhasa_linguistic",
+            "seahelm_linguistic",
             f"lindsea_pragmatics_scalar_implicatures_{language}",
             f"lindsea_pragmatics_scalar_implicatures_{subset}_{language}",
         ],

helm/benchmark/run_specs/speech_disorder_audio_run_specs.py

@@ -0,0 +1,169 @@
+from typing import List, Optional
+from helm.benchmark.adaptation.adapter_spec import (
+    ADAPT_GENERATION_MULTIMODAL,
+    ADAPT_MULTIPLE_CHOICE_JOINT_MULTIMODAL,
+    AdapterSpec,
+)
+from helm.benchmark.metrics.common_metric_specs import (
+    get_basic_generation_metric_specs,
+    get_basic_metric_specs,
+    get_multiple_choice_classification_metric_specs,
+)
+from helm.benchmark.metrics.metric import MetricSpec
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+def audio_classification_metric_specs() -> List[MetricSpec]:
+    return get_multiple_choice_classification_metric_specs() + get_basic_metric_specs(
+        ["exact_match", "quasi_exact_match"]
+    )
+
+
+def _get_multiple_choice_joint_adapter_spec(
+    input_noun: Optional[str],
+    output_noun: str,
+    max_train_instances: int = 0,
+    num_outputs: int = 1,
+) -> AdapterSpec:
+    return AdapterSpec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT_MULTIMODAL,
+        global_prefix="",
+        instructions="Answer the multiple choice question by just giving the letter of the correct answer "
+        "and nothing else.",
+        input_prefix=f"{input_noun}: " if input_noun is not None else "",
+        input_suffix="\n",
+        output_prefix=f"{output_noun}: ",
+        output_suffix="\n",
+        instance_prefix="\n",
+        max_train_instances=max_train_instances,
+        num_outputs=num_outputs,
+        max_tokens=1,
+        stop_sequences=["\n"],
+        temperature=0.0,
+        random=None,
+    )
+
+
+def _get_generation_adapter_spec(
+    max_tokens: int,
+    instructions: str = "",
+    max_train_instances: int = 0,
+    temperature: float = 0.0,
+    stop_sequences: Optional[List[str]] = None,
+) -> AdapterSpec:
+    return AdapterSpec(
+        method=ADAPT_GENERATION_MULTIMODAL,
+        instructions=instructions,
+        input_prefix="",
+        input_suffix="",
+        output_prefix="",
+        output_suffix="",
+        instance_prefix="",
+        max_train_instances=max_train_instances,
+        num_outputs=1,
+        max_tokens=max_tokens,
+        temperature=temperature,
+        stop_sequences=stop_sequences if stop_sequences is not None else [],
+    )
+
+
+@run_spec_function("ultra_suite_classification")
+def get_ultra_suite_classification_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_classification_scenario.UltraSuiteClassificationScenario",  # noqa: E501
+    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
+        input_noun=None, output_noun="Answer", max_train_instances=0
+    )
+    metric_specs: List[MetricSpec] = audio_classification_metric_specs()
+    run_spec_name: str = "ultra_suite_classification"
+    return RunSpec(
+        name=f"{run_spec_name}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+@run_spec_function("ultra_suite_classification_breakdown")
+def get_ultra_suite_disorder_breakdown_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_disorder_breakdown_scenario.UltraSuiteDisorderBreakdownScenario",  # noqa: E501
+    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
+        input_noun=None, output_noun="Answer", max_train_instances=0
+    )
+    metric_specs: List[MetricSpec] = audio_classification_metric_specs()
+    run_spec_name: str = "ultra_suite_classification_breakdown"
+    return RunSpec(
+        name=f"{run_spec_name}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+# Makes the model transcribe the child's speech into text without assuming what the child is supposed to say
+# if the transcription matches the prompt, then it is classified as typically developing
+# otherwise, it is classified as having a speech disorder
+@run_spec_function("ultra_suite_asr_classification")
+def get_ultra_suite_asr_classification_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_classification.UltraSuiteASRClassificationScenario",  # noqa: E501
+    )
+    adapter_spec = _get_generation_adapter_spec(
+        instructions="""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording is provided to you, typically consisting of a speech prompt from a pathologist followed by a child's repetition. Based on your expertise transcribe the child's speech into text. Do not make any assumptions about the words the child is expected to say. Only transcribe based on the words that the child actually says. Only respond with the text transcription, no other text or commentary.""",  # noqa: E501
+        max_tokens=10,
+    )
+    metric_specs: List[MetricSpec] = audio_classification_metric_specs()
+    run_spec_name: str = "ultra_suite_asr_classification"
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+# Makes the model transcribe the child's speech into text and is allowed to assume what the child is supposed to say
+@run_spec_function("ultra_suite_asr_transcription")
+def get_ultra_suite_asr_transcription_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_asr_classification.UltraSuiteASRClassificationScenario",  # noqa: E501
+    )
+    adapter_spec = _get_generation_adapter_spec(
+        instructions="""You are a highly experienced Speech-Language Pathologist (SLP). An audio recording will be provided, typically consisting of a speech prompt from a pathologist followed by a child's repetition. Based on your expertise transcribe the child's speech into text. Try to understand what the child is expected to say. And only respond with the transcription of the child's speech. Not the pathologist's prompt or any other commentary. Only respond with the text transcription, no other text, commentary or punctuations.""",  # noqa: E501
+        max_tokens=50,
+    )
+    metric_specs: List[MetricSpec] = get_basic_generation_metric_specs(["wer_score", "mer_score", "wip_score"])
+    run_spec_name: str = "ultra_suite_asr_transcription"
+    return RunSpec(
+        name=run_spec_name,
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )
+
+
+@run_spec_function("ultra_suite_disorder_symptoms")
+def get_ultra_suite_disorder_symptoms_run_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.audio_language.ultra_suite_disorder_symptoms_scenario.UltraSuiteDisorderSymptomsScenario",  # noqa: E501
+    )
+    adapter_spec: AdapterSpec = _get_multiple_choice_joint_adapter_spec(
+        input_noun=None, output_noun="Answer", max_train_instances=0
+    )
+    metric_specs: List[MetricSpec] = audio_classification_metric_specs()
+    run_spec_name: str = "ultra_suite_disorder_symptoms"
+    return RunSpec(
+        name=f"{run_spec_name}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=metric_specs,
+        groups=[run_spec_name],
+    )