crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/metrics/summac/model_summac.py

@@ -10,7 +10,7 @@ import numpy.typing as npt
 import torch
 import os
 import json
-from . import utils_misc
+from helm.benchmark.metrics.summac import utils_misc
 
 model_map = {
     "snli-base": {"model_card": "boychaboy/SNLI_roberta-base", "entailment_idx": 0, "contradiction_idx": 2},
@@ -50,7 +50,6 @@ class SummaCImager:
     def __init__(
         self, model_name="mnli", granularity="paragraph", use_cache=True, max_doc_sents=100, device="cuda", **kwargs
     ):
-
         self.grans = granularity.split("-")
 
         assert (
@@ -154,7 +153,6 @@ class SummaCImager:
             for j in range(N_gen)
         ]
         for batch in utils_misc.batcher(dataset, batch_size=20):
-
             if self.model_name == "decomp":
                 batch_evids, batch_conts, batch_neuts = [], [], []
                 batch_json = [{"premise": d["premise"], "hypothesis": d["hypothesis"]} for d in batch]
@@ -171,19 +169,18 @@ class SummaCImager:
                 batch_tokens = self.tokenizer.batch_encode_plus(
                     list(zip(batch_prems, batch_hypos)),
                     padding=True,
-                    truncation=True,
+                    truncation="only_first",
                     max_length=self.max_input_length,
                     return_tensors="pt",
-                    truncation_strategy="only_first",
                 )
                 batch_tokens = {k: v.to(self.device) for k, v in batch_tokens.items()}
                 with torch.no_grad():
                     model_outputs = self.model(**batch_tokens)
 
                 batch_probs = torch.nn.functional.softmax(model_outputs["logits"], dim=-1)
-                batch_evids = batch_probs[:, self.entailment_idx].tolist()
-                batch_conts = batch_probs[:, self.contradiction_idx].tolist()
-                batch_neuts = batch_probs[:, self.neutral_idx].tolist()
+                batch_evids = batch_probs[:, self.entailment_idx].tolist()  # type: ignore
+                batch_conts = batch_probs[:, self.contradiction_idx].tolist()  # type: ignore
+                batch_neuts = batch_probs[:, self.neutral_idx].tolist()  # type: ignore
 
             for b, evid, cont, neut in zip(batch, batch_evids, batch_conts, batch_neuts):
                 image[0, b["doc_i"], b["gen_i"]] = evid
@@ -222,7 +219,7 @@ class SummaCConv(torch.nn.Module):
         imager_load_cache=True,
         agg="mean",
         norm_histo=False,
-        **kwargs
+        **kwargs,
     ):
         # `bins` should be `even%d` or `percentiles`
         assert nli_labels in ["e", "c", "n", "ec", "en", "cn", "ecn"], "Unrecognized nli_labels argument %s" % (
@@ -243,7 +240,7 @@ class SummaCConv(torch.nn.Module):
 
         if "even" in bins:
             n_bins = int(bins.replace("even", ""))
-            self.bins = list(np.arange(0, 1, 1 / n_bins)) + [1.0]
+            self.bins = np.arange(0, 1, 1 / n_bins).tolist() + [1.0]
         elif bins == "percentile":
             self.bins = [
                 0.0,
@@ -408,7 +405,7 @@ class SummaCZS:
         use_con=True,
         imager_load_cache=True,
         device="cuda",
-        **kwargs
+        **kwargs,
     ):
         assert op2 in ["min", "mean", "max"], "Unrecognized `op2`"
         assert op1 in ["max", "mean", "min"], "Unrecognized `op1`"

helm/benchmark/metrics/summarization_critique_metrics.py

@@ -2,10 +2,10 @@ from typing import Dict, List
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from .metric import Metric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
 from helm.common.critique_request import CritiqueTaskTemplate, CritiqueQuestionTemplate, CritiqueRequest, QuestionType
 
 

helm/benchmark/metrics/summarization_metrics.py

@@ -12,16 +12,20 @@ from collections import defaultdict
 from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.evaluate_reference_metrics import get_rouge_function
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
 from helm.common.optional_dependencies import handle_module_not_found_error
-from .metric import Metric, MetricResult
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .basic_metrics import get_rouge_function
-from .statistic import Stat
-from .summac.model_summac import SummaCZS
-from bert_score import BERTScorer
+from helm.benchmark.metrics.metric import Metric, MetricMetadata, MetricResult
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.summac.model_summac import SummaCZS
+
+try:
+    from bert_score import BERTScorer  # type: ignore
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["summarization"])
 
 
 QAFACTEVAL_URL: str = (
@@ -43,7 +47,14 @@ class SummarizationMetric(Metric):
     4. Faithfulness (SummaC)
     """
 
-    def __init__(self, task: str, device: str = "cpu"):
+    def __init__(
+        self,
+        task: str,
+        language: str = "en",
+        device: str = "cpu",
+        bertscore_model: str = "microsoft/deberta-large-mnli",
+        rescale_with_baseline: bool = True,
+    ):
         self.rouge_fns = {
             "rouge_1": get_rouge_function("rouge1"),
             "rouge_2": get_rouge_function("rouge2"),
@@ -56,9 +67,9 @@ class SummarizationMetric(Metric):
             spacy.cli.download("en_core_web_sm")
 
         try:
-            from summ_eval.data_stats_metric import DataStatsMetric
+            from summ_eval.data_stats_metric import DataStatsMetric  # type: ignore
         except ModuleNotFoundError as e:
-            handle_module_not_found_error(e, ["metrics"])
+            handle_module_not_found_error(e, ["summarization"])
 
         self.data_stats_metric = DataStatsMetric()
         self.task: str = task
@@ -71,7 +82,7 @@ class SummarizationMetric(Metric):
         else:
             self.compute_bertscore = True
             self.bert_scorer = BERTScorer(
-                model_type="microsoft/deberta-large-mnli", lang="en", rescale_with_baseline=True, device=device
+                model_type=bertscore_model, lang=language, rescale_with_baseline=rescale_with_baseline, device=device
             )
             # Need GPU for faithfulness metrics since they are model-based.
             self.compute_faithfulness = True
@@ -229,6 +240,134 @@ class SummarizationMetric(Metric):
 
         return result
 
+    def get_metadata(self):
+        metadata: List[MetricMetadata] = [
+            MetricMetadata(
+                name="QAFactEval",
+                display_name="QAFactEval",
+                description="Faithfulness scores based on the SummaC method of [Laban et al. "
+                "(2022)](https://aclanthology.org/2022.tacl-1.10/).",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="summarization_coverage",
+                display_name="Coverage",
+                description="Extent to which the model-generated summaries are extractive fragments from the source "
+                "document [(Grusky et al., 2018)](https://aclanthology.org/N18-1065/).",
+                lower_is_better=None,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="summarization_density",
+                display_name="Density",
+                description="Extent to which the model-generated summaries are extractive summaries based on the "
+                "source document [(Grusky et al., 2018)](https://aclanthology.org/N18-1065/).",
+                lower_is_better=None,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="summarization_compression",
+                display_name="Compression",
+                description="Extent to which the model-generated summaries are compressed relative to the source "
+                "document [(Grusky et al., 2018)](https://aclanthology.org/N18-1065/).",
+                lower_is_better=None,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="rouge_1",
+                display_name="ROUGE-1",
+                short_display_name="ROUGE-1",
+                description="ROUGE-1",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="rouge-2",
+                display_name="ROUGE-2",
+                short_display_name="ROUGE-2",
+                description="ROUGE-2",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="rouge-l",
+                display_name="ROUGE-L",
+                short_display_name="ROUGE-L",
+                description="ROUGE-L",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+        ]
+        if self.humaneval is not None:
+            metadata.extend(
+                [
+                    MetricMetadata(
+                        name="HumanEval-faithfulness",
+                        display_name="HumanEval-faithfulness",
+                        description="Human evaluation score for faithfulness.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                    MetricMetadata(
+                        name="HumanEval-relevance",
+                        display_name="HumanEval-relevance",
+                        description="Human evaluation score for relevance.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                    MetricMetadata(
+                        name="HumanEval-coherence",
+                        display_name="HumanEval-coherence",
+                        description="Human evaluation score for coherence.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                ]
+            )
+        if self.compute_faithfulness:
+            metadata.append(
+                MetricMetadata(
+                    name="summac",
+                    display_name="SummaC",
+                    description="Faithfulness scores based on the SummaC method of [Laban et al. "
+                    "(2022)](https://aclanthology.org/2022.tacl-1.10/).",
+                    lower_is_better=False,
+                    group="summarization_metrics",
+                )
+            )
+        if self.compute_bertscore:
+            metadata.extend(
+                [
+                    MetricMetadata(
+                        name="BERTScore-P",
+                        display_name="BERTScore (P)",
+                        description="Average BERTScore precision [(Zhang et al., "
+                        "2020)](https://openreview.net/pdf?id=SkeHuCVFDr) between model generation and reference "
+                        "summary.",
+                        lower_is_better=False,
+                        group=None,
+                    ),
+                    MetricMetadata(
+                        name="BERTScore-R",
+                        display_name="BERTScore (R)",
+                        description="Average BERTScore recall [(Zhang et al., 2020)](https://openreview.net/pdf?id=SkeHuCVFDr) "
+                        "between model generation and reference summary.",
+                        lower_is_better=False,
+                        group=None,
+                    ),
+                    MetricMetadata(
+                        name="BERTScore-F",
+                        display_name="BERTScore (F1)",
+                        description="Average BERTScore F1 [(Zhang et al., 2020)](https://openreview.net/pdf?id=SkeHuCVFDr) "
+                        "between model generation and reference summary.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                ]
+            )
+        return metadata
+
 
 def _paired_bootstrap_test(treatment_list: list, control_list: list, nboot: int = 10000):
     """

helm/benchmark/metrics/test_bias_metrics.py

@@ -1,7 +1,11 @@
 from dataclasses import dataclass
 from typing import Callable, List, Optional
 
-from .bias_metrics import BiasMetric
+from helm.benchmark.metrics.bias_metrics import BiasMetric
+from helm.benchmark.metrics.nltk_helper import install_nltk_resources
+
+
+install_nltk_resources()
 
 
 @dataclass(frozen=True)

helm/benchmark/metrics/test_classification_metrics.py

@@ -7,7 +7,7 @@ from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.classification_metrics import ClassificationMetric
 from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.scenarios.scenario import Input, Instance, Output, Reference, CORRECT_TAG
-from helm.common.request import Request, RequestResult, Sequence
+from helm.common.request import Request, RequestResult, GeneratedOutput
 
 
 class _Option(NamedTuple):
@@ -28,19 +28,24 @@ def _request_state(prediction: str, options: List[_Option]):
         output_mapping=None,
         request=Request(model="openai/text-davinci-002", model_deployment="openai/text-davinci-002"),
         result=RequestResult(
-            success=True, embedding=[], completions=[Sequence(text=prediction, logprob=0.0, tokens=[])], cached=False
+            success=True,
+            embedding=[],
+            completions=[GeneratedOutput(text=prediction, logprob=0.0, tokens=[])],
+            cached=False,
         ),
         num_train_instances=0,
         prompt_truncated=False,
     )
 
 
-def assert_stats_equal(actual_stats: List[Stat], expected_values: Dict[str, float]):
-    actual_values = {stat.name.name: stat.mean for stat in actual_stats}
-    assert actual_values == approx(expected_values)
+def get_stat_value(stats: List[Stat], stat_name: str):
+    for stat in stats:
+        if stat.name.name == stat_name:
+            return stat.mean
+    raise ValueError(f"No stat with name {stat_name}")
 
 
-def _expected_stats(all_classes_counts: Dict[str, Dict[str, int]]):
+def compute_stats(all_classes_counts: Dict[str, Dict[str, int]]):
     micro_counts: Dict[str, int] = defaultdict(int)
     for class_counts in all_classes_counts.values():
         for key, class_count in class_counts.items():
@@ -49,102 +54,172 @@ def _expected_stats(all_classes_counts: Dict[str, Dict[str, int]]):
     micro_recall = micro_counts["tp"] / (micro_counts["tp"] + micro_counts["fn"])
     micro_f1 = 2 * (micro_precision * micro_recall) / (micro_precision + micro_recall)
 
-    class_f1: List[float] = []
-    for class_counts in all_classes_counts.values():
+    class_f1: Dict[str, float] = {}
+    for class_name, class_counts in all_classes_counts.items():
         class_precision = class_counts["tp"] / (class_counts["tp"] + class_counts["fp"])
         class_recall = class_counts["tp"] / (class_counts["tp"] + class_counts["fn"])
-        class_f1.append(2 * (class_precision * class_recall) / (class_precision + class_recall))
-    macro_f1 = sum(class_f1) / len(class_f1)
-
-    return {
-        "classification_micro_f1": micro_f1,
-        "classification_macro_f1": macro_f1,
+        class_f1[class_name] = 2 * (class_precision * class_recall) / (class_precision + class_recall)
+    macro_f1 = sum(class_f1.values()) / len(class_f1)
+    class_name_to_support = {
+        class_name: class_counts["tp"] + class_counts["fn"] for class_name, class_counts in all_classes_counts.items()
     }
+    weighted_f1 = sum(class_f1[class_name] * support for class_name, support in class_name_to_support.items()) / sum(
+        support for support in class_name_to_support.values()
+    )
 
+    stats = {
+        "macro_f1": macro_f1,
+        "micro_f1": micro_f1,
+        "weighted_f1": weighted_f1,
+    }
+    for class_name, class_f1_score in class_f1.items():
+        stats[f"{class_name}_f1"] = class_f1_score
+    return stats
 
-def test_evaluate_instances_binary_generation():
-    metric = ClassificationMetric(delimiter=None)
 
+def test_evaluate_instances_default_parameters():
     request_states = [
         _request_state("yes", [_Option("yes", True)]),
+        _request_state("yes ", [_Option("yes", True)]),
+        _request_state("yeS", [_Option("yes", True)]),
+        _request_state("yes", [_Option("no", True)]),
+        _request_state("no", [_Option("yes", True)]),
+        _request_state("no", [_Option("no", True)]),
+        _request_state("invalid", [_Option("no", True)]),
+    ]
+
+    expected_stats = compute_stats(
+        {
+            "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
+            "no": {"tp": 1, "fp": 1, "tn": 3, "fn": 2},
+        }
+    )
+
+    actual_stats = ClassificationMetric().evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+
+
+def test_evaluate_instances_yes_and_no():
+    labels = ["yes", "no"]
+    request_states = [
         _request_state("yes", [_Option("yes", True)]),
-        _request_state("yes", [_Option("yes", True)]),
+        _request_state("yes ", [_Option("yes", True)]),
+        _request_state("yeS", [_Option("yes", True)]),
         _request_state("yes", [_Option("no", True)]),
         _request_state("no", [_Option("yes", True)]),
         _request_state("no", [_Option("no", True)]),
         _request_state("invalid", [_Option("no", True)]),
     ]
 
-    assert_stats_equal(
-        metric.evaluate_instances(request_states),
-        _expected_stats(
-            {
-                "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
-                "no": {"tp": 1, "fp": 1, "tn": 3, "fn": 2},
-            }
-        ),
+    expected_stats = compute_stats(
+        {
+            "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
+            "no": {"tp": 1, "fp": 1, "tn": 3, "fn": 2},
+        }
     )
 
+    actual_stats = ClassificationMetric(
+        scores=["f1"], averages=["macro", "micro", "weighted", None], labels=labels
+    ).evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+    actual_weighted_f1 = get_stat_value(actual_stats, "classification_weighted_f1")
+    assert actual_weighted_f1 == approx(expected_stats["weighted_f1"])
+    actual_yes_f1 = get_stat_value(actual_stats, "classification_yes_f1")
+    assert actual_yes_f1 == approx(expected_stats["yes_f1"])
+    actual_no_f1 = get_stat_value(actual_stats, "classification_no_f1")
+    assert actual_no_f1 == approx(expected_stats["no_f1"])
+
 
 def test_evaluate_instances_multi_class():
-    # Note: no "a" because it would get filtered out by normalize_text()
-    metric = ClassificationMetric(delimiter=None)
+    labels = ["a", "b", "c"]
 
-    def _options(correct: str):
-        return [_Option(text, text == correct) for text in ["d", "b", "c"]]
+    def _gold_label(correct: str):
+        return [_Option(text, text == correct) for text in labels]
 
     request_states = [
-        _request_state("d", _options("d")),
-        _request_state("d", _options("d")),
-        _request_state("d", _options("d")),
-        _request_state("d", _options("b")),
-        _request_state("b", _options("b")),
-        _request_state("b", _options("b")),
-        _request_state("b", _options("c")),
-        _request_state("c", _options("d")),
-        _request_state("c", _options("c")),
-        _request_state("invalid", _options("c")),
+        _request_state("a", _gold_label("a")),
+        _request_state("a", _gold_label("a")),
+        _request_state("a", _gold_label("a")),
+        _request_state("a", _gold_label("b")),
+        _request_state("b", _gold_label("b")),
+        _request_state("b", _gold_label("b")),
+        _request_state("b", _gold_label("c")),
+        _request_state("c", _gold_label("a")),
+        _request_state("c", _gold_label("c")),
+        _request_state("invalid", _gold_label("c")),
     ]
-    assert_stats_equal(
-        metric.evaluate_instances(request_states),
-        _expected_stats(
-            {
-                "d": {"tp": 3, "fp": 1, "tn": 5, "fn": 1},
-                "b": {"tp": 2, "fp": 1, "tn": 6, "fn": 1},
-                "c": {"tp": 1, "fp": 1, "tn": 6, "fn": 2},
-            }
-        ),
+
+    expected_stats = compute_stats(
+        {
+            "a": {"tp": 3, "fp": 1, "tn": 5, "fn": 1},
+            "b": {"tp": 2, "fp": 1, "tn": 6, "fn": 1},
+            "c": {"tp": 1, "fp": 1, "tn": 6, "fn": 2},
+        }
     )
 
+    actual_stats = ClassificationMetric(
+        scores=["f1"], averages=["macro", "micro", "weighted", None], labels=labels
+    ).evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+    actual_weighted_f1 = get_stat_value(actual_stats, "classification_weighted_f1")
+    assert actual_weighted_f1 == approx(expected_stats["weighted_f1"])
+    actual_a_f1 = get_stat_value(actual_stats, "classification_a_f1")
+    assert actual_a_f1 == approx(expected_stats["a_f1"])
+    actual_b_f1 = get_stat_value(actual_stats, "classification_b_f1")
+    assert actual_b_f1 == approx(expected_stats["b_f1"])
+    actual_c_f1 = get_stat_value(actual_stats, "classification_c_f1")
+    assert actual_c_f1 == approx(expected_stats["c_f1"])
+
 
 def test_evaluate_instances_multilabel():
-    # Note: no "a" because it would get filtered out by normalize_text()
-    metric = ClassificationMetric(delimiter=",")
+    labels = ["a", "b", "c"]
 
-    def _options(correct: List[str]):
-        return [_Option(text, text in correct) for text in ["d", "b", "c"]]
+    def _gold_labels(correct: List[str]):
+        return [_Option(text, text in correct) for text in labels]
 
     request_states = [
-        _request_state("d,b", _options(["d", "b"])),
-        _request_state("d,b", _options(["d", "c"])),
-        _request_state("d", _options(["d"])),
-        _request_state("c", _options(["b"])),
-        _request_state("b", _options(["b", "c"])),
-        _request_state("d,b", _options(["c"])),
-        _request_state("d,c", _options(["d"])),
-        _request_state("d,b,c", _options(["d", "b", "c"])),
+        _request_state("a,b", _gold_labels(["a", "b"])),
+        _request_state("a,b", _gold_labels(["a", "c"])),
+        _request_state("a", _gold_labels(["a"])),
+        _request_state("c", _gold_labels(["b"])),
+        _request_state("b", _gold_labels(["b", "c"])),
+        _request_state("a,b", _gold_labels(["c"])),
+        _request_state("a,c", _gold_labels(["a"])),
+        _request_state("a,b,c", _gold_labels(["a", "b", "c"])),
         _request_state("", []),
         _request_state("n/a", []),
-        _request_state("invalid", _options(["c"])),
+        _request_state("invalid", _gold_labels(["c"])),
     ]
 
-    assert_stats_equal(
-        metric.evaluate_instances(request_states),
-        _expected_stats(
-            {
-                "d": {"tp": 5, "fp": 1, "tn": 5, "fn": 0},
-                "b": {"tp": 3, "fp": 2, "tn": 5, "fn": 1},
-                "c": {"tp": 1, "fp": 2, "tn": 4, "fn": 4},
-            }
-        ),
+    expected_stats = compute_stats(
+        {
+            "a": {"tp": 5, "fp": 1, "tn": 5, "fn": 0},
+            "b": {"tp": 3, "fp": 2, "tn": 5, "fn": 1},
+            "c": {"tp": 1, "fp": 2, "tn": 4, "fn": 4},
+        }
     )
+
+    actual_stats = ClassificationMetric(
+        scores=["f1"], averages=["macro", "micro", "weighted", None], labels=labels, delimiter=","
+    ).evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+    actual_weighted_f1 = get_stat_value(actual_stats, "classification_weighted_f1")
+    assert actual_weighted_f1 == approx(expected_stats["weighted_f1"])
+    actual_a_f1 = get_stat_value(actual_stats, "classification_a_f1")
+    assert actual_a_f1 == approx(expected_stats["a_f1"])
+    actual_b_f1 = get_stat_value(actual_stats, "classification_b_f1")
+    assert actual_b_f1 == approx(expected_stats["b_f1"])
+    actual_c_f1 = get_stat_value(actual_stats, "classification_c_f1")
+    assert actual_c_f1 == approx(expected_stats["c_f1"])

helm/benchmark/metrics/test_disinformation_metrics.py

@@ -0,0 +1,78 @@
+# Test metrics
+from typing import List
+
+import numpy as np
+import pytest
+from helm.benchmark.metrics.disinformation_metrics import _monte_carlo_entropy, _self_bleu
+from helm.common.request import GeneratedOutput, Token
+
+# Test tokens
+_TEST_1_TOKENS: List[Token] = [
+    Token("This", logprob=-0.25),
+    Token("is", logprob=-0.25),
+    Token("a", logprob=-0.25),
+    Token("test", logprob=-0.25),
+]
+_TEST_2_TOKENS: List[Token] = [
+    Token("This", logprob=-0.25),
+    Token("is", logprob=-0.25),
+    Token("another", logprob=-0.5),
+    Token("test", logprob=-0.25),
+]
+_TEST_EMPTY_TOKENS: List[Token] = []
+test_empty_str_tokens: List[Token] = [
+    Token("", logprob=0),
+]
+
+# Test Sequences (two standard, one with an empty token, and one with no tokens)
+_TEST_1 = GeneratedOutput(text="This is a test", logprob=-1, tokens=_TEST_1_TOKENS)
+_TEST_2 = GeneratedOutput(text="This is another test", logprob=-1.25, tokens=_TEST_2_TOKENS)
+_TEST_EMPTY = GeneratedOutput(text="", logprob=-float("nan"), tokens=_TEST_EMPTY_TOKENS)
+_TEST_EMPTY_STR = GeneratedOutput(text="", logprob=0, tokens=test_empty_str_tokens)
+
+
+# Test Self-BLEU
+def test_self_bleu_with_self():
+    score = _self_bleu([_TEST_1, _TEST_1])
+    assert score == pytest.approx(100)
+
+
+def test_self_blue_with_other():
+    score = _self_bleu([_TEST_1, _TEST_2])
+    assert 0 < score < 100
+
+
+def test_self_blue_one_sequence():
+    score = _self_bleu([_TEST_1])
+    assert score == 0
+
+
+def test_self_blue_one_full_one_empty():
+    score = _self_bleu([_TEST_1, _TEST_EMPTY_STR])
+    assert score == 0
+
+
+# Test MC Entropy
+def test_mc_entropy_with_self():
+    score = _monte_carlo_entropy([_TEST_1, _TEST_1])
+    assert score == pytest.approx(-_TEST_1.logprob)
+
+
+def test_mc_entropy_with_other():
+    score = _monte_carlo_entropy([_TEST_1, _TEST_2])
+    assert score == pytest.approx(-(_TEST_1.logprob + _TEST_2.logprob) / 2)
+
+
+def test_mc_entropy_one_sequence():
+    score = _monte_carlo_entropy([_TEST_1])
+    assert score == -_TEST_1.logprob
+
+
+def test_mc_entropy_one_full_one_empty():
+    score = _monte_carlo_entropy([_TEST_EMPTY_STR])
+    assert score == _TEST_EMPTY_STR.logprob
+
+
+def test_mc_entropy_with_no_tokens():
+    score = _monte_carlo_entropy([_TEST_EMPTY])
+    assert np.isnan(score)