crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/run_specs/arabic_run_specs.py

@@ -0,0 +1,197 @@
+"""Run specs for Arabic leaderboard
+
+EXPERIMENTAL: Run specs here may have future reverse incompatible changes."""
+
+from helm.benchmark.adaptation.adapter_spec import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec, get_generation_adapter_spec
+from helm.benchmark.annotation.annotator import AnnotatorSpec
+from helm.benchmark.metrics.common_metric_specs import get_basic_metric_specs, get_exact_match_metric_specs
+from helm.benchmark.metrics.metric import MetricSpec
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+_ARABIC_REFERENCE_PREFIX_CHARACTERS = ["أ", "ب", "ج", "د", "هـ"]
+_ARABIC_OUTPUT_MAPPING_PATTERN = "(أ|ب|ج|د|هـ)"
+
+
+@run_spec_function("arabic_mmlu")
+def get_arabic_mmlu_spec(subset: str) -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.arabic_mmlu_scenario.ArabicMMLUScenario", args={"subset": subset}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+        output_mapping_pattern=_ARABIC_OUTPUT_MAPPING_PATTERN,
+    )
+
+    return RunSpec(
+        name=f"arabic_mmlu:subset={subset}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["arabic_mmlu"],
+    )
+
+
+@run_spec_function("alghafa")
+def get_alghafa_spec(subset: str) -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.alghafa_scenario.AlGhafaScenario", args={"subset": subset}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+        output_mapping_pattern=_ARABIC_OUTPUT_MAPPING_PATTERN,
+    )
+
+    return RunSpec(
+        name=f"alghafa:subset={subset}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["alghafa", f"alghafa_{subset}"],
+    )
+
+
+@run_spec_function("aratrust")
+def get_aratrust_spec(category: str) -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.aratrust_scenario.AraTrustScenario",
+        args={"category": category},
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة: أ، ب أو ج",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+    )
+
+    return RunSpec(
+        name=f"aratrust:category={category}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["aratrust"],
+    )
+
+
+@run_spec_function("alrage")
+def get_alrage_spec() -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.alrage_scenario.ALRAGEScenario")
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions="بناءً على السياقات المقترحة التالية، اجب عن السؤال التالي",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+    )
+
+    annotator_specs = [AnnotatorSpec(class_name="helm.benchmark.annotation.alrage_annotator.ALRAGEAnnotator")]
+
+    metric_specs = [
+        MetricSpec(class_name="helm.benchmark.metrics.alrage_metric.ALRAGEMetric")
+    ] + get_basic_metric_specs([])
+
+    return RunSpec(
+        name="alrage",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        annotators=annotator_specs,
+        metric_specs=metric_specs,
+        groups=["alrage"],
+    )
+
+
+@run_spec_function("madinah_qa")
+def get_madinah_qa_spec(subset: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.madinah_qa_scenario.MadinahQAScenario", args={"subset": subset}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+        output_mapping_pattern=_ARABIC_OUTPUT_MAPPING_PATTERN,
+    )
+
+    return RunSpec(
+        name=f"madinah_qa:subset={subset}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["madinah_qa"],
+    )
+
+
+@run_spec_function("mbzuai_human_translated_arabic_mmlu")
+def get_arabic_mmmlu_spec(subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.mbzuai_human_translated_arabic_mmlu.MBZUAIHumanTranslatedArabicMMLUScenario",
+        args={"subject": subject},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+        output_mapping_pattern=_ARABIC_OUTPUT_MAPPING_PATTERN,
+    )
+
+    return RunSpec(
+        name=f"mbzuai_human_translated_arabic_mmlu:subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["mbzuai_human_translated_arabic_mmlu"],
+    )
+
+
+@run_spec_function("arabic_exams")
+def get_arabic_exams_spec(subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.arabic_exams_scenario.ArabicEXAMSScenario",
+        args={"subject": subject},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+        output_mapping_pattern=_ARABIC_OUTPUT_MAPPING_PATTERN,
+    )
+
+    return RunSpec(
+        name=f"arabic_exams:subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["arabic_exams"],
+    )

helm/benchmark/run_specs/bluex_run_specs.py

@@ -0,0 +1,40 @@
+from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("bluex")
+def get_bluex_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bluex_scenario.BLUEXScenario", args={})
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="""
+        Escolha a alternativa correta para as questões de vestibulares (responda apenas com a letra).
+        Exemplo de Pergunta com a resposta:
+        Em um romance narrado em primeira pessoa, o narrador participa dos acontecimentos da trama,
+        relatando suas próprias experiências e sentimentos. Qual alternativa apresenta essa característica?
+
+        (A) Narrador onisciente que conhece os pensamentos de todas as personagens.
+        (B) Narrador que descreve os fatos de forma imparcial, sem envolvimento emocional.
+        (C) Narrador-personagem que vivencia e relata os eventos da história.
+        (D) Narrador observador que apenas registra as ações visíveis.
+        (E) Narrador em segunda pessoa que se dirige constantemente ao leitor.
+
+        Resposta correta: C
+
+        A partir disso, responda:
+        """,
+        input_noun="Pergunta",
+        output_noun="Resposta",
+    )
+
+    return RunSpec(
+        name="bluex",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["bluex"],
+    )

helm/benchmark/run_specs/classic_run_specs.py

@@ -35,7 +35,6 @@ from helm.benchmark.metrics.common_metric_specs import (
     get_f1_metric_specs,
     get_generative_harms_metric_specs,
     get_language_modeling_metric_specs,
-    get_numeracy_metric_specs,
     get_open_ended_generation_metric_specs,
     get_summarization_metric_specs,
     get_basic_generation_metric_specs,
@@ -381,58 +380,6 @@ def get_raft_spec(subset: str) -> RunSpec:
     )
 
 
-@run_spec_function("numeracy")
-def get_numeracy_spec(
-    relation_type: str = "linear", mode: str = "function", seed: str = "0", run_solver: str = "False"
-) -> RunSpec:
-    from helm.benchmark.scenarios.numeracy_scenario import get_numeracy_adapter_spec, RELTYPE_INFO
-
-    run_solver_bool: bool = True if run_solver.lower() == "true" else False
-    del run_solver
-    random_seed = int(seed)
-    scenario_spec = ScenarioSpec(
-        class_name="helm.benchmark.scenarios.numeracy_scenario.NumeracyScenario",
-        args={"seed": random_seed, "relation_type": relation_type, "mode": mode},
-    )
-
-    if mode in ["example", "standard"]:
-        # Test a model's ability to impute datapoints for a given (example or randomly sampled) relation.
-        adapter_args: Dict[str, Any] = {
-            "max_train_instances": 100,
-            "max_eval_instances": 100,
-            "dim": RELTYPE_INFO[relation_type].num_variables + 1,
-        }
-    elif mode == "function":
-        # Test a model's ability to impute datapoints for randomly sampled relations
-        # (resampled for each evaluation point).
-        adapter_args = {
-            "instructions": "",
-            "max_train_instances": 0,  # Turn off general version of `function` mode because it doesn't cleanly
-            # capture a higher-order version of this task / is a little convoluted
-            # for models, currently.
-            # (In the general version, the model sees other relations of the same class,
-            # and needs to impute a datapoint for the last one. Presumably, inferring
-            # the class - eg. the degree of the relation - would help.)
-            "max_eval_instances": 1000,
-            "dim": RELTYPE_INFO[relation_type].num_variables + 1,
-            "instance_prefix": "\n\n",
-        }
-    else:
-        raise ValueError(f"Invalid mode: {mode}")
-
-    adapter_spec = get_numeracy_adapter_spec(**adapter_args)  # Construct the AdapterSpec using a helper function.
-    # `get_numeracy_adapter_spec` is defined in numeracy_scenario.py
-    # because it is used within the scenario to construct the instances themselves.
-
-    return RunSpec(
-        name=f"numeracy:relation_type={relation_type},mode={mode}",
-        scenario_spec=scenario_spec,
-        adapter_spec=adapter_spec,
-        metric_specs=get_numeracy_metric_specs(run_solver_bool),
-        groups=["numeracy"],
-    )
-
-
 @run_spec_function("boolq")
 def get_boolq_spec(only_contrast=False) -> RunSpec:
     scenario_spec = ScenarioSpec(
@@ -806,12 +753,12 @@ def get_xsum_sampled_summarization_spec(temperature: float = 0.3, device: str =
     )
 
     return RunSpec(
-        name=f"summarization_xsum:temperature={temperature},device={device}",
+        name=f"summarization_xsum_sampled:temperature={temperature},device={device}",
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_summarization_metric_specs({"task": "summarization_xsum_sampled", "device": device})
         + get_generative_harms_metric_specs(),
-        groups=["summarization_xsum"],
+        groups=["summarization_xsum_sampled"],
     )
 
 

helm/benchmark/run_specs/codeinsights_run_specs.py

@@ -0,0 +1,192 @@
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+from helm.benchmark.adaptation.common_adapter_specs import get_generation_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_basic_metric_specs
+from helm.benchmark.metrics.codeinsights_metric_specs import (
+    get_functional_correctness_metric_specs,
+    get_comprehensive_code_evaluation_metric_specs,
+    get_edge_case_metric_specs,
+    get_code_efficiency_metric_specs,
+)
+
+
+@run_spec_function("codeinsights_correct_code")
+def get_codeinsights_correct_code_run_spec(tpr: float = 0.0, num_testcases: int = 1) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.codeinsights_correct_code_scenario.CodeInsightsCorrectCodeScenario",
+        args={"num_testcases": num_testcases},
+    )
+
+    instruction = (
+        "You are a skilled C++ programmer working on a foundational programming course assignment. "
+        "Your task is to write correct, efficient C++ code that solves the given problem. "
+        "Write clean, well-structured code that follows good programming practices. "
+        "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template."
+        "DO NOT reproduce the template part as the generated code would be inserted to the template,"
+        "and make sure the code is compatible with the Unit Test Input"
+        "Ensure your code is correct, efficient, includes any class definition when needed, and handles all edge cases properly."
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=instruction,
+        output_noun="Your code",
+        stop_sequences=[],
+        max_tokens=4000,
+        temperature=tpr,
+    )
+
+    return RunSpec(
+        name=f"codeinsights_correct_code:temperature={adapter_spec.temperature},num_testcases={num_testcases}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_functional_correctness_metric_specs() + get_basic_metric_specs([]),
+        groups=["codeinsights", "codeinsights_correct_code"],
+    )
+
+
+@run_spec_function("codeinsights_student_coding")
+def get_codeinsights_student_coding_run_spec(tpr: float = 0.0, num_testcases: int = 1) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.codeinsights_student_coding_scenario.CodeInsightsStudentCodingScenario",
+        args={"num_testcases": num_testcases},
+    )
+
+    instruction = (
+        "You are the same student who wrote the three examples below in your foundational C++ course. "
+        "Mimic exactly your personal coding style, conventions, and level of proficiency—"
+        "do not over‐optimize or introduce unfamiliar patterns. "
+        "Include the same sort of formatting, variable names, and minor imperfections you demonstrated. "
+        "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template."
+        "DO NOT reproduce the template part as the generated code would be inserted to the template,"
+        "and make sure the code is compatible with the Unit Test Input"
+        "Ensure your code includes any class definition when needed."
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=instruction,
+        output_noun="Your code",
+        stop_sequences=[],
+        max_tokens=4000,
+        temperature=tpr,
+    )
+
+    return RunSpec(
+        name=f"codeinsights_student_coding:temperature={adapter_spec.temperature},num_testcases={num_testcases}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_comprehensive_code_evaluation_metric_specs() + get_basic_metric_specs([]),
+        groups=["codeinsights", "codeinsights_student_coding"],
+    )
+
+
+@run_spec_function("codeinsights_student_mistake")
+def get_codeinsights_student_mistake_run_spec(tpr: float = 0.0, num_testcases: int = 1) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.codeinsights_student_mistake_scenario.CodeInsightsStudentMistakeScenario",
+        args={"num_testcases": num_testcases},
+    )
+
+    instruction = (
+        "You are a C++ student with a consistent personal style, conventions, and proficiency level.\n"
+        "Your task is to attempt the target problem **but introduce realistic mistake** you would typically make—"
+        "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template."
+        "DO NOT reproduce the template part as the generated code would be inserted to the template,"
+        "and make sure the code is compatible with the Unit Test Input"
+        "Ensure your code is includes any class definition when needed."
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=instruction,
+        output_noun="Your code",
+        stop_sequences=[],
+        max_tokens=4000,
+        temperature=tpr,
+    )
+
+    return RunSpec(
+        name=f"codeinsights_student_mistake:temperature={adapter_spec.temperature},num_testcases={num_testcases}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_comprehensive_code_evaluation_metric_specs() + get_basic_metric_specs([]),
+        groups=["codeinsights", "codeinsights_student_mistake"],
+    )
+
+
+@run_spec_function("codeinsights_code_efficiency")
+def get_codeinsights_code_efficiency_run_spec(tpr: float = 0.0, num_testcases: int = 1) -> RunSpec:
+    """
+    Run specification for code efficiency evaluation scenario.
+
+    This scenario evaluates whether LLM-generated code has similar runtime efficiency
+    as the original student code. It focuses on problems where both solutions are
+    functionally correct and measures runtime performance alignment.
+
+    Requires C++ compiler (g++) to be available for actual compilation and execution.
+    """
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.codeinsights_code_efficiency_scenario.CodeInsightsCodeEfficiencyScenario",
+        args={"num_testcases": num_testcases},
+    )
+
+    instruction = (
+        "You are the same student who wrote the three examples below in your foundational C++ course. "
+        "Mimic exactly your personal coding style, conventions, and make sure to generate a correct code. "
+        "Do not over-optimize or introduce unfamiliar patterns. If the code is correct but inefficient, "
+        "imitate the inefficiency. "
+        "If the student writes efficiently, write efficiently too. "
+        "Include the same sort of formatting, variable names, and minor imperfections you demonstrated. "
+        "Provide ONLY your C++ implementation following the given template, where the answer will replace the {{ STUDENT_ANSWER }} block in the template."
+        "DO NOT reproduce the template part as the generated code would be inserted to the template,"
+        "and make sure the code is compatible with the Unit Test Input"
+        "Ensure your code is correct, includes any class definition when needed, and handles all edge cases properly."
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=instruction,
+        output_noun="Your code",
+        stop_sequences=[],
+        max_tokens=4000,
+        temperature=tpr,
+    )
+
+    return RunSpec(
+        name=f"codeinsights_code_efficiency:temperature={adapter_spec.temperature},num_testcases={num_testcases}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_code_efficiency_metric_specs(
+            num_runtime_runs=5,  # Run each solution 5 times for averaging
+            timeout_seconds=10,  # 10 second timeout per execution
+        )
+        + get_basic_metric_specs([]),
+        groups=["codeinsights", "codeinsights_code_efficiency"],
+    )
+
+
+@run_spec_function("codeinsights_edge_case")
+def get_codeinsights_edge_case_run_spec(tpr: float = 0.0, num_testcases: int = 1) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.codeinsights_edge_case_scenario.CodeInsightsEdgeCaseScenario",
+        args={"num_testcases": num_testcases},
+    )
+
+    instruction = (
+        "You are a student studying C++ with a consistent personal style, conventions, and proficiency level.\n"
+        "Your task is to identify which test case you would likely to fail for a given question with unit tests.\n"
+        "Respond only with integer of the unittest number\n\n"
+    )
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions=instruction,
+        output_noun="Your code",
+        stop_sequences=[],
+        max_tokens=4000,
+        temperature=tpr,
+    )
+
+    return RunSpec(
+        name=f"codeinsights_edge_case:temperature={adapter_spec.temperature},num_testcases={num_testcases}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_edge_case_metric_specs() + get_basic_metric_specs([]),
+        groups=["codeinsights", "codeinsights_edge_case"],
+    )

helm/benchmark/run_specs/healthqa_br_run_specs.py

@@ -0,0 +1,40 @@
+from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_MULTIPLE_CHOICE_JOINT
+from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec
+from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.run_spec import RunSpec, run_spec_function
+from helm.benchmark.scenarios.scenario import ScenarioSpec
+
+
+@run_spec_function("healthqa_br")
+def get_healthqa_br_spec() -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.healthqa_br_scenario.HEALTHQA_BR_Scenario", args={}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="""
+        Escolha a alternativa correta para as questões de medicina (responda apenas com a letra).
+        Exemplo de Pergunta com a resposta:
+        Qual dos seguintes órgãos é responsável pela produção da insulina no corpo humano?
+        A) Fígado
+        B) Rins
+        C) Pâncreas
+        D) Baço
+        E) Coração
+
+        Resposta correta: C
+
+        A partir disso, responda:
+        """,
+        input_noun="Pergunta",
+        output_noun="Resposta",
+    )
+
+    return RunSpec(
+        name="healthqa_br",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["healthqa_br"],
+    )

helm/benchmark/run_specs/heim_run_specs.py

@@ -60,7 +60,9 @@ def get_core_heim_metric_specs() -> List[MetricSpec]:
             class_name="helm.benchmark.metrics.image_generation.fractal_dimension_metric.FractalDimensionMetric",
             args={},
         ),
-        MetricSpec(class_name="helm.benchmark.metrics.image_generation.nsfw_metrics.NSFWMetric", args={}),
+        # Disabled due to keras issue.
+        # See: https://github.com/stanford-crfm/helm/issues/3741#issuecomment-3109478877
+        # MetricSpec(class_name="helm.benchmark.metrics.image_generation.nsfw_metrics.NSFWMetric", args={}),
         MetricSpec(class_name="helm.benchmark.metrics.image_generation.nudity_metrics.NudityMetric", args={}),
         MetricSpec(class_name="helm.benchmark.metrics.image_generation.watermark_metrics.WatermarkMetric", args={}),
     ] + get_basic_metric_specs(names=[])