crfm-helm 0.5.7__py3-none-any.whl → 0.5.8__py3-none-any.whl

helm/benchmark/presentation/taxonomy_info.py

@@ -0,0 +1,20 @@
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass(frozen=True)
+class TaxonomyInfo:
+    # Task (e.g., question answering)
+    task: Optional[str] = None
+
+    # Domain - genre (e.g., Wikipedia)
+    what: Optional[str] = None
+
+    # Domain - when it was written (e.g., 2010s)
+    when: Optional[str] = None
+
+    # Domain - demographics (e.g., web users)
+    who: Optional[str] = None
+
+    # Language (e.g., English)
+    language: Optional[str] = None

helm/benchmark/run_expander.py

@@ -1484,6 +1484,8 @@ class OutputFormatInstructions(RunExpander):
                 instructions = "Answer with only a single letter. Do not include a period in your answer."
             elif self.scenario == "mcqa_only_last_question":
                 instructions = "Answer only the last question with only a single letter."
+            elif self.scenario == "arabic_mcqa":
+                instructions = "اكتب حرف الإجابة فقط، دون أي إضافات أخرى."
             else:
                 instructions = "Answer with only a single letter."
         elif run_spec.adapter_spec.method == ADAPT_GENERATION:
@@ -1525,6 +1527,8 @@ class OutputFormatInstructions(RunExpander):
                     "Answer only the last question with a short answer. "
                     "Avoid extra, unnecessary information in the answer."
                 )
+            elif self.scenario == "arabic_mcqa":
+                instructions = "اكتب حرف الإجابة فقط، دون أي إضافات أخرى."
             else:
                 raise ValueError(f"Unknown scenario {self.scenario}")
         elif run_spec.adapter_spec.method == ADAPT_MULTIPLE_CHOICE_JOINT_CHAIN_OF_THOUGHT:

helm/benchmark/run_specs/arabic_run_specs.py

@@ -4,25 +4,35 @@ EXPERIMENTAL: Run specs here may have future reverse incompatible changes."""
 
 from helm.benchmark.adaptation.adapter_spec import ADAPT_MULTIPLE_CHOICE_JOINT
 from helm.benchmark.adaptation.common_adapter_specs import get_multiple_choice_adapter_spec, get_generation_adapter_spec
-from helm.benchmark.metrics.common_metric_specs import get_exact_match_metric_specs
+from helm.benchmark.annotation.annotator import AnnotatorSpec
+from helm.benchmark.metrics.common_metric_specs import get_basic_metric_specs, get_exact_match_metric_specs
+from helm.benchmark.metrics.metric import MetricSpec
 from helm.benchmark.run_spec import RunSpec, run_spec_function
 from helm.benchmark.scenarios.scenario import ScenarioSpec
 
 
+_ARABIC_REFERENCE_PREFIX_CHARACTERS = ["أ", "ب", "ج", "د", "هـ"]
+
+
 @run_spec_function("arabic_mmlu")
-def get_arabic_mmlu_spec() -> RunSpec:
+def get_arabic_mmlu_spec(subset: str) -> RunSpec:
     """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.arabic_mmlu_scenario.ArabicMMLUScenario")
+
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.arabic_mmlu_scenario.ArabicMMLUScenario", args={"subset": subset}
+    )
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
-        instructions="The following are multiple choice questions. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
-        input_noun="Question",
-        output_noun="Answer",
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
     )
 
     return RunSpec(
-        name="arabic_mmlu",
+        name=f"arabic_mmlu:subset={subset}",
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs(),
@@ -39,9 +49,11 @@ def get_alghafa_spec(subset: str) -> RunSpec:
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
-        instructions="The following are multiple choice questions. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
-        input_noun="Question",
-        output_noun="Answer",
+        instructions="الأسئلة التالية هي أسئلة متعددة الإختيارات مع الجواب الصحيح",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
     )
 
     return RunSpec(
@@ -54,20 +66,126 @@ def get_alghafa_spec(subset: str) -> RunSpec:
 
 
 @run_spec_function("aratrust")
-def get_aratrust_spec() -> RunSpec:
+def get_aratrust_spec(category: str) -> RunSpec:
     """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.aratrust_scenario.AraTrustScenario")
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.aratrust_scenario.AraTrustScenario",
+        args={"category": category},
+    )
 
     adapter_spec = get_generation_adapter_spec(
-        instructions="The following are multiple choice questions. Answer the last question. Respond only with only a single letter corresponding to your choice.",  # noqa: E501
-        input_noun="Question",
-        output_noun="Answer",
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة: أ، ب أو ج",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
     )
 
     return RunSpec(
-        name="aratrust",
+        name=f"aratrust:category={category}",
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_exact_match_metric_specs(),
         groups=["aratrust"],
     )
+
+
+@run_spec_function("alrage")
+def get_alrage_spec() -> RunSpec:
+    """EXPERIMENTAL: This run spec here may have future reverse incompatible changes."""
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.alrage_scenario.ALRAGEScenario")
+
+    adapter_spec = get_generation_adapter_spec(
+        instructions="بناءً على السياقات المقترحة التالية، اجب عن السؤال التالي",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+    )
+
+    annotator_specs = [AnnotatorSpec(class_name="helm.benchmark.annotation.alrage_annotator.ALRAGEAnnotator")]
+
+    metric_specs = [
+        MetricSpec(class_name="helm.benchmark.metrics.alrage_metric.ALRAGEMetric")
+    ] + get_basic_metric_specs([])
+
+    return RunSpec(
+        name="alrage",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        annotators=annotator_specs,
+        metric_specs=metric_specs,
+        groups=["alrage"],
+    )
+
+
+@run_spec_function("madinah_qa")
+def get_madinah_qa_spec(subset: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.madinah_qa_scenario.MadinahQAScenario", args={"subset": subset}
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+    )
+
+    return RunSpec(
+        name=f"madinah_qa:subset={subset}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["madinah_qa"],
+    )
+
+
+@run_spec_function("mbzuai_human_translated_arabic_mmlu")
+def get_arabic_mmmlu_spec(subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.mbzuai_human_translated_arabic_mmlu.MBZUAIHumanTranslatedArabicMMLUScenario",
+        args={"subject": subject},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+    )
+
+    return RunSpec(
+        name=f"mbzuai_human_translated_arabic_mmlu:subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["mbzuai_human_translated_arabic_mmlu"],
+    )
+
+
+@run_spec_function("arabic_exams")
+def get_arabic_exams_spec(subject: str) -> RunSpec:
+    scenario_spec = ScenarioSpec(
+        class_name="helm.benchmark.scenarios.arabic_exams_scenario.ArabicEXAMSScenario",
+        args={"subject": subject},
+    )
+
+    adapter_spec = get_multiple_choice_adapter_spec(
+        method=ADAPT_MULTIPLE_CHOICE_JOINT,
+        instructions="السؤال التالي هو سؤال متعدد الإختيارات. اختر الإجابة الصحيحة",  # noqa: E501
+        input_noun="السؤال",
+        output_noun="الإجابة",
+        max_tokens=100,
+        reference_prefix_characters=_ARABIC_REFERENCE_PREFIX_CHARACTERS,
+    )
+
+    return RunSpec(
+        name=f"arabic_exams:subject={subject}",
+        scenario_spec=scenario_spec,
+        adapter_spec=adapter_spec,
+        metric_specs=get_exact_match_metric_specs(),
+        groups=["arabic_exams"],
+    )

helm/benchmark/run_specs/bluex_run_specs.py

@@ -7,7 +7,7 @@ from helm.benchmark.scenarios.scenario import ScenarioSpec
 
 @run_spec_function("bluex")
 def get_bluex_spec() -> RunSpec:
-    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bluex_scenario.BLUEX_Scenario", args={})
+    scenario_spec = ScenarioSpec(class_name="helm.benchmark.scenarios.bluex_scenario.BLUEXScenario", args={})
 
     adapter_spec = get_multiple_choice_adapter_spec(
         method=ADAPT_MULTIPLE_CHOICE_JOINT,

helm/benchmark/run_specs/classic_run_specs.py

@@ -753,12 +753,12 @@ def get_xsum_sampled_summarization_spec(temperature: float = 0.3, device: str =
     )
 
     return RunSpec(
-        name=f"summarization_xsum:temperature={temperature},device={device}",
+        name=f"summarization_xsum_sampled:temperature={temperature},device={device}",
         scenario_spec=scenario_spec,
         adapter_spec=adapter_spec,
         metric_specs=get_summarization_metric_specs({"task": "summarization_xsum_sampled", "device": device})
         + get_generative_harms_metric_specs(),
-        groups=["summarization_xsum"],
+        groups=["summarization_xsum_sampled"],
     )
 
 

helm/benchmark/run_specs/long_context_run_specs.py

@@ -39,12 +39,12 @@ def _get_long_context_multiple_choice_adapter_spec(max_tokens: int) -> AdapterSp
         method=ADAPT_MULTIPLE_CHOICE_JOINT,
         global_prefix="",
         global_suffix="",
-        instructions="Read the passage and answer the following question. Respond with only a single letter corresponding to your choice.",  # noqa: E501
+        instructions="Read the passage and answer the following question. Respond with only a single letter corresponding to your choice. Do not include a period in your answer.\n\n",  # noqa: E501
         input_prefix="",
         input_suffix="\n",
         reference_prefix="A. ",
         reference_suffix="\n",
-        output_prefix="",
+        output_prefix="\nAnswer the question above based on the passage. Respond with only a single letter corresponding to your choice. Do not include a period in your answer.\n",  # noqa: E501
         output_suffix="",
         instance_prefix="",
         max_train_instances=0,

helm/benchmark/run_specs/medhelm/benchmark_config.py

@@ -0,0 +1,219 @@
+import yaml
+import json
+import re
+
+from dataclasses import dataclass
+from typing import Any, Dict, List, Union
+from abc import ABC
+
+from helm.benchmark.annotation.annotator import AnnotatorSpec
+from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo
+from helm.benchmark.metrics.metric import MetricSpec
+from helm.benchmark.metrics.common_metric_specs import (
+    get_basic_metric_specs,
+    get_exact_match_metric_specs,
+    get_summarization_metric_specs,
+)
+from helm.common.gpu_utils import get_torch_device_name
+
+
+SUMMARIZATION_METRICS = {
+    "rouge_1",
+    "rouge_2",
+    "rouge_l",
+    "BERTScore-P",
+    "BERTScore-R",
+    "BERTScore-F",
+}
+
+
+@dataclass(frozen=True)
+class MetricConfig(ABC):
+    """Base class for all metric configurations"""
+
+    name: str
+
+
+@dataclass(frozen=True)
+class SimpleMetricConfig(MetricConfig):
+    """Configuration for simple string-based metrics like 'exact_match'"""
+
+    pass
+
+
+@dataclass(frozen=True)
+class JuryMetricConfig(MetricConfig):
+    """Configuration for jury-based metrics with multiple judges"""
+
+    prompt_file: str
+    judges: List[AnnotatorModelInfo]
+
+
+@dataclass(frozen=True)
+class BenchmarkConfig:
+    """
+    A benchmark configuration is an immutable data structure that holds
+    the configuration for a specific benchmark, including prompt, dataset and metric
+    """
+
+    name: str
+    """Name of the benchmark"""
+
+    description: str
+    """Description of the benchmark"""
+
+    prompt_file: str
+    """Path to the prompt file. This prompt will be used for all instances of the benchmark."""
+
+    dataset_file: str
+    """Path to the dataset file. This dataset will be used to populate the context in the prompt."""
+
+    main_metric: Union[SimpleMetricConfig, JuryMetricConfig]
+    """The main metric for the benchmark"""
+
+    metrics: List[Union[SimpleMetricConfig, JuryMetricConfig]]
+    """List of structured metric configurations for the benchmark"""
+
+    max_tokens: int = 1024
+    """Maximum number of tokens to generate in the response"""
+
+    def get_metric_specs(self) -> List[MetricSpec]:
+        """Get the metric specifications for the benchmark"""
+        metric_specs: List[MetricSpec] = []
+        summarization = False
+        for metric in self.metrics:
+            if metric.name == "exact_match":
+                metric_specs.extend(get_exact_match_metric_specs())
+
+            elif metric.name == "jury_score":
+                if not isinstance(metric, JuryMetricConfig):
+                    raise AssertionError("Metric 'jury_score' must be a JuryMetricConfig")
+                annotator_models = {judge.model_deployment: judge for judge in metric.judges}
+                metric_specs.append(
+                    MetricSpec(
+                        class_name="helm.benchmark.metrics.llm_jury_metrics.LLMJuryMetric",
+                        args={
+                            "metric_name": "jury_score",
+                            "scenario_name": self.name,
+                            "annotator_models": annotator_models,
+                        },
+                    )
+                )
+                if len(self.metrics) == 1:
+                    metric_specs.extend(get_basic_metric_specs([]))
+
+            elif metric.name in SUMMARIZATION_METRICS:
+                if not summarization:
+                    summarization = True
+                    metric_args = {
+                        "task": self.name,
+                        "device": get_torch_device_name(),
+                        "bertscore_model": "distilbert-base-uncased",
+                        "rescale_with_baseline": False,
+                    }
+                    metric_specs.extend(get_summarization_metric_specs(metric_args))
+            else:
+                raise ValueError(f"Unknown metric name: {metric.name}")
+        return metric_specs
+
+    def _get_annotation_criteria(self, prompt_template: str) -> Dict[str, List[str]]:
+        criteria_tag = re.compile(r"<rubric_criteria>\s*(\{.*?\})\s*</rubric_criteria>", re.DOTALL)
+        m = criteria_tag.search(prompt_template)
+        if not m:
+            raise ValueError("No <rubric_criteria>{...}</rubric_criteria> block found in prompt_template.")
+        raw = json.loads(m.group(1))
+        # normalize to Dict[str, Set[str]]
+        return {k: list(v) for k, v in raw.items()}
+
+    def get_annotator_specs(self) -> List[AnnotatorSpec]:
+        """Convert jury metrics to AnnotatorSpec objects"""
+        annotator_specs = []
+        # return annotator_specs
+        for metric in self.metrics:
+            if isinstance(metric, JuryMetricConfig):
+                with open(metric.prompt_file, "r") as f:
+                    prompt_template = f.read()
+                annotator_models = {judge.model_deployment: judge for judge in metric.judges}
+                annotator_criteria = self._get_annotation_criteria(prompt_template)
+                # Create a generic annotator spec - you may need to customize the class_name
+                # based on your specific use case
+                annotator_specs.append(
+                    AnnotatorSpec(
+                        class_name="helm.benchmark.annotation.model_as_judge.LLMAsJuryAnnotator",
+                        args={
+                            "name": self.name,
+                            "prompt_template": prompt_template,
+                            "annotation_criteria": annotator_criteria,
+                            "annotator_models": annotator_models,
+                        },
+                    )
+                )
+
+        return annotator_specs
+
+
+def _convert_metrics(raw_metrics: List[Dict[str, Any]]) -> List[MetricConfig]:
+    """
+    Convert raw metrics from YAML into structured MetricConfig objects.
+    """
+    converted_metrics: List[MetricConfig] = []
+
+    for metric in raw_metrics:
+        if not isinstance(metric, dict) or "name" not in metric:
+            raise ValueError(
+                f"Invalid metric format: {metric}. Each metric must be a dict with at least a 'name' field."
+            )
+
+        metric_name = metric["name"]
+
+        if metric_name == "jury_score":
+            if "prompt_file" not in metric or "judges" not in metric:
+                raise ValueError(f"jury_score metric requires 'prompt_file' and 'judges': {metric}")
+
+            judges = [
+                AnnotatorModelInfo(
+                    model_name=j["model_name"],
+                    model_deployment=j["name"],
+                )
+                for j in metric["judges"]
+            ]
+
+            converted_metrics.append(
+                JuryMetricConfig(name=metric_name, prompt_file=metric["prompt_file"], judges=judges)
+            )
+        else:
+            converted_metrics.append(SimpleMetricConfig(name=metric_name))
+
+    return converted_metrics
+
+
+def _structure_benchmark_config(data: Dict[str, Any], cls) -> BenchmarkConfig:
+    """
+    Custom structure function for BenchmarkConfig that handles metrics conversion
+    """
+    if "metrics" in data:
+        data = data.copy()  # Don't modify the original
+        raw_metrics = data["metrics"]
+        data["metrics"] = _convert_metrics(raw_metrics)
+        data["main_metric"] = data["metrics"][0]
+    else:
+        raise ValueError("No metrics specified.")
+
+    return BenchmarkConfig(
+        name=data["name"],
+        description=data["description"],
+        prompt_file=data["prompt_file"],
+        dataset_file=data["dataset_file"],
+        main_metric=data["main_metric"],
+        metrics=data["metrics"],
+        max_tokens=data.get("max_tokens", 1024),
+    )
+
+
+def get_benchmark_config_from_path(path: str) -> BenchmarkConfig:
+    """Load and parse benchmark configuration from YAML file"""
+    with open(path) as f:
+        config = yaml.safe_load(f)
+
+    benchmark_config = _structure_benchmark_config(config, BenchmarkConfig)
+    return benchmark_config