crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/metrics/test_classification_metrics.py

@@ -38,12 +38,14 @@ def _request_state(prediction: str, options: List[_Option]):
     )
 
 
-def assert_stats_equal(actual_stats: List[Stat], expected_values: Dict[str, float]):
-    actual_values = {stat.name.name: stat.mean for stat in actual_stats}
-    assert actual_values == approx(expected_values)
+def get_stat_value(stats: List[Stat], stat_name: str):
+    for stat in stats:
+        if stat.name.name == stat_name:
+            return stat.mean
+    raise ValueError(f"No stat with name {stat_name}")
 
 
-def _expected_stats(all_classes_counts: Dict[str, Dict[str, int]]):
+def compute_stats(all_classes_counts: Dict[str, Dict[str, int]]):
     micro_counts: Dict[str, int] = defaultdict(int)
     for class_counts in all_classes_counts.values():
         for key, class_count in class_counts.items():
@@ -52,102 +54,172 @@ def _expected_stats(all_classes_counts: Dict[str, Dict[str, int]]):
     micro_recall = micro_counts["tp"] / (micro_counts["tp"] + micro_counts["fn"])
     micro_f1 = 2 * (micro_precision * micro_recall) / (micro_precision + micro_recall)
 
-    class_f1: List[float] = []
-    for class_counts in all_classes_counts.values():
+    class_f1: Dict[str, float] = {}
+    for class_name, class_counts in all_classes_counts.items():
         class_precision = class_counts["tp"] / (class_counts["tp"] + class_counts["fp"])
         class_recall = class_counts["tp"] / (class_counts["tp"] + class_counts["fn"])
-        class_f1.append(2 * (class_precision * class_recall) / (class_precision + class_recall))
-    macro_f1 = sum(class_f1) / len(class_f1)
-
-    return {
-        "classification_micro_f1": micro_f1,
-        "classification_macro_f1": macro_f1,
+        class_f1[class_name] = 2 * (class_precision * class_recall) / (class_precision + class_recall)
+    macro_f1 = sum(class_f1.values()) / len(class_f1)
+    class_name_to_support = {
+        class_name: class_counts["tp"] + class_counts["fn"] for class_name, class_counts in all_classes_counts.items()
     }
+    weighted_f1 = sum(class_f1[class_name] * support for class_name, support in class_name_to_support.items()) / sum(
+        support for support in class_name_to_support.values()
+    )
 
+    stats = {
+        "macro_f1": macro_f1,
+        "micro_f1": micro_f1,
+        "weighted_f1": weighted_f1,
+    }
+    for class_name, class_f1_score in class_f1.items():
+        stats[f"{class_name}_f1"] = class_f1_score
+    return stats
 
-def test_evaluate_instances_binary_generation():
-    metric = ClassificationMetric(delimiter=None)
 
+def test_evaluate_instances_default_parameters():
     request_states = [
         _request_state("yes", [_Option("yes", True)]),
+        _request_state("yes ", [_Option("yes", True)]),
+        _request_state("yeS", [_Option("yes", True)]),
+        _request_state("yes", [_Option("no", True)]),
+        _request_state("no", [_Option("yes", True)]),
+        _request_state("no", [_Option("no", True)]),
+        _request_state("invalid", [_Option("no", True)]),
+    ]
+
+    expected_stats = compute_stats(
+        {
+            "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
+            "no": {"tp": 1, "fp": 1, "tn": 3, "fn": 2},
+        }
+    )
+
+    actual_stats = ClassificationMetric().evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+
+
+def test_evaluate_instances_yes_and_no():
+    labels = ["yes", "no"]
+    request_states = [
         _request_state("yes", [_Option("yes", True)]),
-        _request_state("yes", [_Option("yes", True)]),
+        _request_state("yes ", [_Option("yes", True)]),
+        _request_state("yeS", [_Option("yes", True)]),
         _request_state("yes", [_Option("no", True)]),
         _request_state("no", [_Option("yes", True)]),
         _request_state("no", [_Option("no", True)]),
         _request_state("invalid", [_Option("no", True)]),
     ]
 
-    assert_stats_equal(
-        metric.evaluate_instances(request_states, ""),
-        _expected_stats(
-            {
-                "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
-                "no": {"tp": 1, "fp": 1, "tn": 3, "fn": 2},
-            }
-        ),
+    expected_stats = compute_stats(
+        {
+            "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
+            "no": {"tp": 1, "fp": 1, "tn": 3, "fn": 2},
+        }
     )
 
+    actual_stats = ClassificationMetric(
+        scores=["f1"], averages=["macro", "micro", "weighted", None], labels=labels
+    ).evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+    actual_weighted_f1 = get_stat_value(actual_stats, "classification_weighted_f1")
+    assert actual_weighted_f1 == approx(expected_stats["weighted_f1"])
+    actual_yes_f1 = get_stat_value(actual_stats, "classification_yes_f1")
+    assert actual_yes_f1 == approx(expected_stats["yes_f1"])
+    actual_no_f1 = get_stat_value(actual_stats, "classification_no_f1")
+    assert actual_no_f1 == approx(expected_stats["no_f1"])
+
 
 def test_evaluate_instances_multi_class():
-    # Note: no "a" because it would get filtered out by normalize_text()
-    metric = ClassificationMetric(delimiter=None)
+    labels = ["a", "b", "c"]
 
-    def _options(correct: str):
-        return [_Option(text, text == correct) for text in ["d", "b", "c"]]
+    def _gold_label(correct: str):
+        return [_Option(text, text == correct) for text in labels]
 
     request_states = [
-        _request_state("d", _options("d")),
-        _request_state("d", _options("d")),
-        _request_state("d", _options("d")),
-        _request_state("d", _options("b")),
-        _request_state("b", _options("b")),
-        _request_state("b", _options("b")),
-        _request_state("b", _options("c")),
-        _request_state("c", _options("d")),
-        _request_state("c", _options("c")),
-        _request_state("invalid", _options("c")),
+        _request_state("a", _gold_label("a")),
+        _request_state("a", _gold_label("a")),
+        _request_state("a", _gold_label("a")),
+        _request_state("a", _gold_label("b")),
+        _request_state("b", _gold_label("b")),
+        _request_state("b", _gold_label("b")),
+        _request_state("b", _gold_label("c")),
+        _request_state("c", _gold_label("a")),
+        _request_state("c", _gold_label("c")),
+        _request_state("invalid", _gold_label("c")),
     ]
-    assert_stats_equal(
-        metric.evaluate_instances(request_states, ""),
-        _expected_stats(
-            {
-                "d": {"tp": 3, "fp": 1, "tn": 5, "fn": 1},
-                "b": {"tp": 2, "fp": 1, "tn": 6, "fn": 1},
-                "c": {"tp": 1, "fp": 1, "tn": 6, "fn": 2},
-            }
-        ),
+
+    expected_stats = compute_stats(
+        {
+            "a": {"tp": 3, "fp": 1, "tn": 5, "fn": 1},
+            "b": {"tp": 2, "fp": 1, "tn": 6, "fn": 1},
+            "c": {"tp": 1, "fp": 1, "tn": 6, "fn": 2},
+        }
     )
 
+    actual_stats = ClassificationMetric(
+        scores=["f1"], averages=["macro", "micro", "weighted", None], labels=labels
+    ).evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+    actual_weighted_f1 = get_stat_value(actual_stats, "classification_weighted_f1")
+    assert actual_weighted_f1 == approx(expected_stats["weighted_f1"])
+    actual_a_f1 = get_stat_value(actual_stats, "classification_a_f1")
+    assert actual_a_f1 == approx(expected_stats["a_f1"])
+    actual_b_f1 = get_stat_value(actual_stats, "classification_b_f1")
+    assert actual_b_f1 == approx(expected_stats["b_f1"])
+    actual_c_f1 = get_stat_value(actual_stats, "classification_c_f1")
+    assert actual_c_f1 == approx(expected_stats["c_f1"])
+
 
 def test_evaluate_instances_multilabel():
-    # Note: no "a" because it would get filtered out by normalize_text()
-    metric = ClassificationMetric(delimiter=",")
+    labels = ["a", "b", "c"]
 
-    def _options(correct: List[str]):
-        return [_Option(text, text in correct) for text in ["d", "b", "c"]]
+    def _gold_labels(correct: List[str]):
+        return [_Option(text, text in correct) for text in labels]
 
     request_states = [
-        _request_state("d,b", _options(["d", "b"])),
-        _request_state("d,b", _options(["d", "c"])),
-        _request_state("d", _options(["d"])),
-        _request_state("c", _options(["b"])),
-        _request_state("b", _options(["b", "c"])),
-        _request_state("d,b", _options(["c"])),
-        _request_state("d,c", _options(["d"])),
-        _request_state("d,b,c", _options(["d", "b", "c"])),
+        _request_state("a,b", _gold_labels(["a", "b"])),
+        _request_state("a,b", _gold_labels(["a", "c"])),
+        _request_state("a", _gold_labels(["a"])),
+        _request_state("c", _gold_labels(["b"])),
+        _request_state("b", _gold_labels(["b", "c"])),
+        _request_state("a,b", _gold_labels(["c"])),
+        _request_state("a,c", _gold_labels(["a"])),
+        _request_state("a,b,c", _gold_labels(["a", "b", "c"])),
         _request_state("", []),
         _request_state("n/a", []),
-        _request_state("invalid", _options(["c"])),
+        _request_state("invalid", _gold_labels(["c"])),
     ]
 
-    assert_stats_equal(
-        metric.evaluate_instances(request_states, ""),
-        _expected_stats(
-            {
-                "d": {"tp": 5, "fp": 1, "tn": 5, "fn": 0},
-                "b": {"tp": 3, "fp": 2, "tn": 5, "fn": 1},
-                "c": {"tp": 1, "fp": 2, "tn": 4, "fn": 4},
-            }
-        ),
+    expected_stats = compute_stats(
+        {
+            "a": {"tp": 5, "fp": 1, "tn": 5, "fn": 0},
+            "b": {"tp": 3, "fp": 2, "tn": 5, "fn": 1},
+            "c": {"tp": 1, "fp": 2, "tn": 4, "fn": 4},
+        }
     )
+
+    actual_stats = ClassificationMetric(
+        scores=["f1"], averages=["macro", "micro", "weighted", None], labels=labels, delimiter=","
+    ).evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+    actual_weighted_f1 = get_stat_value(actual_stats, "classification_weighted_f1")
+    assert actual_weighted_f1 == approx(expected_stats["weighted_f1"])
+    actual_a_f1 = get_stat_value(actual_stats, "classification_a_f1")
+    assert actual_a_f1 == approx(expected_stats["a_f1"])
+    actual_b_f1 = get_stat_value(actual_stats, "classification_b_f1")
+    assert actual_b_f1 == approx(expected_stats["b_f1"])
+    actual_c_f1 = get_stat_value(actual_stats, "classification_c_f1")
+    assert actual_c_f1 == approx(expected_stats["c_f1"])

helm/benchmark/metrics/test_evaluate_reference_metrics.py

@@ -1,4 +1,7 @@
+import pytest
 from helm.benchmark.metrics.evaluate_reference_metrics import (
+    bleu_1,
+    chinese_bleu_1,
     exact_match,
     exact_match_indicator,
     final_number_exact_match,
@@ -28,3 +31,15 @@ def test_final_number_exact_match():
     assert final_number_exact_match("34.2", "2") == 0
     assert final_number_exact_match("342", "342.") == 1
     assert final_number_exact_match("3,420", "3420") == 1
+
+
+def test_bleu_1():
+    assert bleu_1("the quick brown fox jumped over the lazy dog", "the fox jumped over the dog") == pytest.approx(
+        0.6065306597126334
+    )
+
+
+def test_chinese_bleu_1():
+    assert chinese_bleu_1(
+        "太祖武皇帝，沛國譙人也，姓曹，諱操，字孟德，漢相國參之後。", "太祖武皇帝，沛國譙人也，漢相國參之後。"
+    ) == pytest.approx(0.5907775139012316)

helm/benchmark/metrics/test_metric.py

@@ -1,5 +1,5 @@
 from helm.common.request import Token
-from .basic_metrics import get_num_bytes, convert_tokens_to_text
+from helm.benchmark.metrics.basic_metrics import get_num_bytes, convert_tokens_to_text
 
 
 def test_get_num_bytes():

helm/benchmark/metrics/test_statistic.py

@@ -3,8 +3,8 @@ from typing import Dict
 import pytest
 import statistics
 
-from .metric_name import MetricName
-from .statistic import Stat, merge_stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat, merge_stat
 
 
 def test_stat_add():

helm/benchmark/metrics/tokens/ai21_token_cost_estimator.py

@@ -1,6 +1,6 @@
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class AI21TokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/auto_token_cost_estimator.py

@@ -2,12 +2,12 @@ from typing import Dict
 
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .ai21_token_cost_estimator import AI21TokenCostEstimator
-from .cohere_token_cost_estimator import CohereTokenCostEstimator
-from .free_token_cost_estimator import FreeTokenCostEstimator
-from .gooseai_token_cost_estimator import GooseAITokenCostEstimator
-from .openai_token_cost_estimator import OpenAITokenCostEstimator
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.ai21_token_cost_estimator import AI21TokenCostEstimator
+from helm.benchmark.metrics.tokens.cohere_token_cost_estimator import CohereTokenCostEstimator
+from helm.benchmark.metrics.tokens.free_token_cost_estimator import FreeTokenCostEstimator
+from helm.benchmark.metrics.tokens.gooseai_token_cost_estimator import GooseAITokenCostEstimator
+from helm.benchmark.metrics.tokens.openai_token_cost_estimator import OpenAITokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class AutoTokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/cohere_token_cost_estimator.py

@@ -1,6 +1,6 @@
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class CohereTokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/free_token_cost_estimator.py

@@ -1,6 +1,6 @@
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class FreeTokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py

@@ -2,7 +2,7 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class GooseAITokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/openai_token_cost_estimator.py

@@ -2,7 +2,7 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.benchmark.window_services.window_service import WindowService
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class OpenAITokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py

@@ -1,7 +1,7 @@
 from unittest.mock import Mock
 
 from helm.common.request import Request
-from .ai21_token_cost_estimator import AI21TokenCostEstimator
+from helm.benchmark.metrics.tokens.ai21_token_cost_estimator import AI21TokenCostEstimator
 
 
 class TestAI21TokenCostEstimator:

helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py

@@ -8,7 +8,7 @@ from helm.common.authentication import Authentication
 from helm.common.request import Request
 from helm.common.tokenization_request import TokenizationRequestResult, TokenizationToken
 from helm.proxy.services.remote_service import RemoteService
-from .openai_token_cost_estimator import OpenAITokenCostEstimator
+from helm.benchmark.metrics.tokens.openai_token_cost_estimator import OpenAITokenCostEstimator
 
 
 class TestOpenAITokenCostEstimator:

helm/benchmark/metrics/toxicity_metrics.py

@@ -6,10 +6,10 @@ from helm.common.hierarchical_logger import hlog
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
-from .metric import Metric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
 
 
 class ToxicityMetric(Metric):

helm/benchmark/metrics/unitxt_metrics.py

@@ -1,9 +1,11 @@
+import numbers
 import re
-from typing import Dict, List
+from typing import Dict, List, Set
 
 from datasets import load_dataset
 import evaluate
 
+from helm.common.general import hlog
 from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats
 from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.metrics.metric_name import MetricName
@@ -16,7 +18,10 @@ class UnitxtMetric(MetricInterface):
 
     def __init__(self, **kwargs):
         super().__init__()
-        dataset_name = ",".join(f"{key}={value}" for key, value in kwargs.items())
+        if len(kwargs) == 1 and "recipe" in kwargs:
+            dataset_name = kwargs["recipe"]
+        else:
+            dataset_name = ",".join(f"{key}={value}" for key, value in kwargs.items())
         self.dataset = load_dataset("unitxt/data", dataset_name, trust_remote_code=True)
 
     def evaluate(
@@ -42,6 +47,7 @@ class UnitxtMetric(MetricInterface):
         )
 
         # Extract instance metrics
+        non_number_instance_metric_names: Set[str] = set()
         per_instance_stats: List[PerInstanceStats] = []
         for request_state, evaluate_result in zip(scenario_state.request_states, evaluate_results):
             instance = request_state.instance
@@ -60,9 +66,15 @@ class UnitxtMetric(MetricInterface):
                 )
                 if isinstance(metric_score, list):
                     for metric_score_element in metric_score:
-                        stat = stat.add(metric_score_element)
+                        if isinstance(metric_score_element, numbers.Number):
+                            stat = stat.add(metric_score_element)
+                        else:
+                            non_number_instance_metric_names.add(metric_name)
                 else:
-                    stat = stat.add(metric_score)
+                    if isinstance(metric_score, numbers.Number):
+                        stat = stat.add(metric_score)
+                    else:
+                        non_number_instance_metric_names.add(metric_name)
                 instance_stats.append(stat)
             assert instance.id
             per_instance_stats.append(
@@ -73,6 +85,11 @@ class UnitxtMetric(MetricInterface):
                     stats=instance_stats,
                 )
             )
+        if non_number_instance_metric_names:
+            hlog(
+                "WARNING: Ignored Unitxt instance metrics because "
+                f"they were not numbers: {non_number_instance_metric_names}"
+            )
 
         # Extract global metrics
         aggregated_stats: List[Stat] = []

helm/benchmark/metrics/vision_language/image_metrics.py

@@ -303,7 +303,10 @@ class AnnotatedImageMetrics(Metric):
         if self._lpips_metric is None:
             with warnings.catch_warnings():
                 warnings.filterwarnings("ignore", category=UserWarning)
-                self._lpips_metric = LearnedPerceptualImagePatchSimilarity(net_type="vgg").to(self._device)
+                # https://lightning.ai/docs/torchmetrics/stable/image/learned_perceptual_image_patch_similarity.html
+                self._lpips_metric = LearnedPerceptualImagePatchSimilarity(net_type="vgg", normalize=True).to(
+                    self._device
+                )
 
         preprocessing = transforms.Compose(
             [
@@ -321,7 +324,7 @@ class AnnotatedImageMetrics(Metric):
 
         # Compute the LPIPS score
         assert self._lpips_metric is not None
-        score: float = self._lpips_metric(img1, img2).detach().item()
+        score: float = 1.0 - self._lpips_metric(img1, img2).detach().item()
         return score
 
     def _calculate_fid(self, act1, act2):
@@ -400,7 +403,8 @@ class AnnotatedImageMetrics(Metric):
 
     def compute_ssim(self, generated_image: np.ndarray, reference_image: np.ndarray) -> float:
         """Compute the Structural Similarity Index (SSIM) between the generated and reference images."""
-        return ssim(generated_image, reference_image)
+        # Add 1 and divide by 2 to get a normalized score between 0 and 1, where 1 is the most similar
+        return (ssim(generated_image, reference_image) + 1) / 2
 
     def compute_edit_sim(self, completion: str, reference: str) -> float:
         # `reference` is the entire remaining book for each instance.

helm/benchmark/metrics/wildbench_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class WildBenchScoreMetric(Metric):
+    """Score metrics for WildBench."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["wildbench"]
+        scores: List[float] = []
+        for annotation_key, annotation_value in annotations.items():
+            if annotation_key.endswith("_score") and annotation_value is not None:
+                scores.append(annotation_value)
+        if not scores:
+            raise ValueError("Could not compute WB Score because all annotators failed.")
+        score = sum(scores) / len(scores)
+        score_rescaled = (score - 1) / 9
+        return [
+            Stat(MetricName("wildbench_score")).add(score),
+            Stat(MetricName("wildbench_score_rescaled")).add(score_rescaled),
+        ]

helm/benchmark/model_metadata_registry.py

@@ -22,6 +22,9 @@ CHATML_MODEL_TAG: str = "CHATML_MODEL_TAG"
 # OpenAI Chat format
 OPENAI_CHATGPT_MODEL_TAG: str = "OPENAI_CHATGPT_MODEL_TAG"
 
+# For NOVA models
+NOVA_MODEL_TAG: str = "NOVA_MODEL_TAG"
+
 # For Anthropic models
 ANTHROPIC_CLAUDE_1_MODEL_TAG: str = "ANTHROPIC_CLAUDE_1_MODEL_TAG"
 ANTHROPIC_CLAUDE_2_MODEL_TAG: str = "ANTHROPIC_CLAUDE_2_MODEL_TAG"
@@ -66,10 +69,18 @@ OPEN_FLAMINGO_MODEL_TAG: str = "OPEN_FLAMINGO_MODEL_TAG"
 LIMITED_FUNCTIONALITY_VLM_TAG: str = "LIMITED_FUNCTIONALITY_VLM_TAG"
 FULL_FUNCTIONALITY_VLM_TAG: str = "FULL_FUNCTIONALITY_VLM_TAG"
 
+# For Audio-langauge models (AudioLMs)
+AUDIO_LANGUAGE_MODEL_TAG: str = "AUDIO_LANGUAGE_MODEL_TAG"
+
 # Deprecated models that are no longer available.
 # These are usually closed API models that have been permanently removed
 DEPRECATED_MODEL_TAG: str = "DEPRECATED_MODEL_TAG"
 
+# Unsupported models.
+# These are models that we have chosen not to support because they are
+# private, stale, non-notable, or difficult to implement.
+UNSUPPORTED_MODEL_TAG: str = "UNSUPPORTED_MODEL_TAG"
+
 # Frozen is set to false as the model_deployment_registry.py file
 # might populate the deployment_names field.
 
@@ -208,6 +219,11 @@ def is_vlm(model_name: str) -> bool:
     return model_has_tag(model_name, VISION_LANGUAGE_MODEL_TAG)
 
 
+def is_audiolm(model_name: str) -> bool:
+    """Returns True if the model is a audio-language model (AudioLM). False otherwise."""
+    return model_has_tag(model_name, AUDIO_LANGUAGE_MODEL_TAG)
+
+
 def get_unknown_model_metadata(helm_model_name: str) -> ModelMetadata:
     """Return placeholder ModelMetadata for an unknown model."""
     return ModelMetadata(

helm/benchmark/presentation/create_plots.py

@@ -14,7 +14,6 @@ from helm.benchmark.config_registry import register_builtin_configs_from_helm_pa
 from helm.common.hierarchical_logger import hlog
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.benchmark.model_metadata_registry import MODEL_NAME_TO_MODEL_METADATA
-from helm.benchmark.presentation.summarize import AGGREGATE_WIN_RATE_COLUMN
 
 try:
     import colorcet
@@ -39,6 +38,7 @@ metric_group_to_label = {
     "Efficiency": f"Inference time (s) {DOWN_ARROW}",
 }
 all_metric_groups = list(metric_group_to_label.keys())
+AGGREGATE_WIN_RATE_COLUMN = 1
 
 
 @dataclass

helm/benchmark/presentation/schema.py

@@ -119,6 +119,9 @@ class MetricGroup(Field):
     hide_win_rates: Optional[bool] = None
     """If set to true, do not compute win rates."""
 
+    aggregation_strategies: Optional[List[str]] = None
+    """List with values in {'win_rate','mean'} that correspond to aggregations"""
+
 
 BY_METRIC = "by_metric"
 BY_GROUP = "by_group"