crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/metrics/kpi_edgar_metrics.py

@@ -6,6 +6,7 @@ import numpy as np
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
+from helm.benchmark.metrics.metric import MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.scenarios.kpi_edgar_scenario import KPIEDGARScenario
@@ -119,3 +120,23 @@ class KPIEdgarMetric(EvaluateInstancesMetric):
 
     def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
         return _compute_stats(_request_states_to_pred_gold_pairs(request_states))
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="word_macro_f1_score",
+                display_name="Word F1 Score",
+                description="Word F1 Score",
+                lower_is_better=None,
+                group=None,
+            ),
+            MetricMetadata(
+                name="adjusted_macro_f1_score",
+                display_name="Adjusted Macro F1 Score",
+                short_display_name="Adjusted Macro F1 Score",
+                description="Entity type classification F1 score, adjusted for partial matches following the KPI-Edgar "
+                "paper, macro-averaged across entity types",
+                lower_is_better=None,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/language_modeling_metrics.py

@@ -11,7 +11,7 @@ from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats, add_context
+from helm.benchmark.metrics.metric import MetricInterface, MetricMetadata, MetricResult, PerInstanceStats, add_context
 from helm.benchmark.metrics.metric_name import MetricContext, MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat, merge_stat
@@ -97,3 +97,15 @@ class LanguageModelingMetric(MetricInterface):
         derived_stats: List[Stat] = []
         derived_stats.extend(compute_perplexity_metrics(stats_dict))
         return derived_stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="num_instances",
+                display_name="# eval",
+                short_display_name=None,
+                description="Number of evaluation instances.",
+                lower_is_better=None,
+                group="general_information",
+            ),
+        ]

helm/benchmark/metrics/live_qa_metrics.py

@@ -2,7 +2,7 @@ from typing import List
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -21,3 +21,15 @@ class LiveQAScoreMetric(Metric):
         assert request_state.annotations
         score = request_state.annotations["live_qa"]["score"]
         return [Stat(MetricName("live_qa_score")).add(score)]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="live_qa_score",
+                display_name="Judge Score",
+                short_display_name=None,
+                description="LLM-as-judge score",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/llm_jury_metrics.py

@@ -3,7 +3,7 @@ from typing import Any, Dict, List
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.annotation.model_as_judge import AnnotatorModelInfo
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -44,3 +44,15 @@ class LLMJuryMetric(Metric):
         return [
             Stat(MetricName(self.metric_name)).add(score),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name=self.metric_name,
+                display_name="Jury Score",
+                short_display_name="Jury Score",
+                description="Measures the average score assigned by an LLM-based jury evaluating task performance.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/lmkt_metric_specs.py

@@ -0,0 +1,12 @@
+from typing import List
+
+from helm.benchmark.metrics.metric import MetricSpec
+
+
+def get_semantic_similarity_metric_specs(similarity_fn_name: str = "cosine") -> List[MetricSpec]:
+    return [
+        MetricSpec(
+            class_name="helm.benchmark.metrics.lmkt_metrics.SemanticSimilarityMetric",
+            args={"similarity_fn_name": similarity_fn_name},
+        ),
+    ]

helm/benchmark/metrics/lmkt_metrics.py

@@ -0,0 +1,47 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+from sentence_transformers import SentenceTransformer
+
+
+class SemanticSimilarityMetric(Metric):
+    """Score metrics for LMKT semantic similarity measurement."""
+
+    def __init__(self, similarity_fn_name: str = "cosine"):
+        """
+        Initialize the SemanticSimilarityMetric with a SentenceTransformer model.
+        :param similarity_fn_name: The name of the similarity function to use.
+        Available options are "dot", "cosine", "manhattan" and "euclidean".
+        """
+        super().__init__()
+
+        self.model = SentenceTransformer("paraphrase-multilingual-MiniLM-L12-v2", similarity_fn_name=similarity_fn_name)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+
+        assert request_state.result
+
+        completions = [c.text for c in request_state.result.completions]
+        completion_embeddings = self.model.encode(completions)
+
+        references = [r.output.text for r in request_state.instance.references if CORRECT_TAG in r.tags]
+        reference_embeddings = self.model.encode(references)
+
+        similarities = self.model.similarity(completion_embeddings, reference_embeddings)
+        avg_similarity = similarities.mean().item()
+
+        return [
+            Stat(MetricName("semantic_similarity")).add(avg_similarity),
+        ]

helm/benchmark/metrics/medcalc_bench_metrics.py

@@ -4,7 +4,7 @@ from datetime import datetime
 from typing import List, Dict, Any
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -122,3 +122,16 @@ class MedCalcBenchMetric(Metric):
         return [
             Stat(MetricName("medcalc_bench_accuracy")).add(exact_match),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="medcalc_bench_accuracy",
+                display_name="MedCalc Accuracy",
+                short_display_name="MedCalc Accuracy",
+                description="Comparison based on category. Exact match for categories risk, severity and diagnosis. "
+                "Check if within range for the other categories.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/medec_metrics.py

@@ -1,12 +1,13 @@
 from typing import List
+import re
+
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
 from helm.common.hierarchical_logger import hlog
-import re
 from helm.benchmark.scenarios.scenario import CORRECT_TAG
 
 
@@ -99,3 +100,25 @@ class MedecMetric(Metric):
             Stat(MetricName("medec_error_flag_accuracy")).add(flag_accuracy),
             Stat(MetricName("medec_error_sentence_accuracy")).add(sentence_accuracy),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="medec_error_flag_accuracy",
+                display_name="Medical Error Flag Accuracy",
+                short_display_name="MedecFlagAcc",
+                description="Measures how accurately the model identifies whether a clinical note contains an error "
+                "(binary classification of correct/incorrect).",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="medec_error_sentence_accuracy",
+                display_name="Medical Error Sentence Accuracy",
+                short_display_name="MedecSentenceAcc",
+                description="Measures how accurately the model identifies the specific erroneous sentence within a "
+                "clinical note.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/melt_toxicity_metric.py

@@ -34,7 +34,7 @@ class VietnameseToxicityMetric(Metric):
     def __repr__(self):
         return "ToxicityMetric()"
 
-    def _get_toxicity_score(self, predictions: Dict) -> List[float]:
+    def _get_toxicity_score(self, predictions: List[Dict]) -> List[float]:
         """Extracts toxicity scores from the predictions.
 
         Args:

helm/benchmark/metrics/metric.py

@@ -86,6 +86,28 @@ class Processor:
         return instance_stats
 
 
+@dataclass(frozen=True)
+class MetricMetadata:
+    name: str
+    """Internal name (usually no spaces, etc.)"""
+
+    display_name: Optional[str] = None
+    """What is displayed to the user"""
+
+    short_display_name: Optional[str] = None
+    """What is displayed to the user (e.g., in a table header)"""
+
+    description: Optional[str] = None
+    """Description of the metric"""
+
+    lower_is_better: Optional[bool] = None
+    """Whether a lower vaue for this metric corresponds to a better model
+    (e.g., False for accuracy, True for perplexity, None for num_trials)"""
+
+    group: Optional[str] = None
+    """Name of the default metric group for this metric"""
+
+
 class MetricInterface(ABC):
     """Interface for all Metrics."""
 
@@ -95,6 +117,9 @@ class MetricInterface(ABC):
     ) -> MetricResult:
         pass
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        raise NotImplementedError()
+
 
 class Metric(MetricInterface, ABC):
     """

helm/benchmark/metrics/mimiciv_billing_code_metrics.py

@@ -1,7 +1,7 @@
 from typing import List
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -94,3 +94,34 @@ class MIMICIVBillingCodeMetric(Metric):
             Stat(MetricName("mimiciv_billing_code_recall")).add(recall),
             Stat(MetricName("mimiciv_billing_code_f1")).add(f1),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="mimiciv_billing_code_precision",
+                display_name="Precision for MIMIC Billing Codes",
+                short_display_name="MIMICBillingPre",
+                description="Measures the proportion of correctly predicted ICD codes among all ICD codes predicted by "
+                "the model.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="mimiciv_billing_code_recall",
+                display_name="Recall for MIMIC Billing Codes",
+                short_display_name="MIMICBillingRec",
+                description="Measures the proportion of correctly predicted ICD codes among all ICD codes present in "
+                "the gold standard.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="mimiciv_billing_code_f1",
+                display_name="F1 Score for MIMIC Billing Codes",
+                short_display_name="MIMICBillingF1",
+                description="Measures the harmonic mean of precision and recall for ICD codes, providing a balanced "
+                "evaluation of the model's performance.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/omni_math_metrics.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, List
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -30,3 +30,15 @@ class OmniMATHMetric(Metric):
         return [
             Stat(MetricName("omni_math_accuracy")).add(score),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="omni_math_accuracy",
+                display_name="Omni-MATH Accuracy",
+                short_display_name="Acc",
+                description="Accuracy of the AI output judged by GPT-4.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+        ]

helm/benchmark/metrics/safety_metrics.py

@@ -5,7 +5,7 @@ from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.basic_metrics import compute_request_state_metrics
 from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -77,3 +77,15 @@ class SafetyScoreMetric(Metric):
             raise SafetyScoreMetricException("Could not compute safety score because all annotators failed.")
         stats.append(safety_score_stat)
         return stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="safety_score",
+                display_name="LM Evaluated Safety score",
+                description="LM Evaluated Safety score (mean of Llama3.1 and GPT4o ratings) representing whether the "
+                "output constitutes a successful refusal of unsafe actions (0 is worst, 1 is best).",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/seahelm_metrics.py

@@ -8,7 +8,7 @@ from sacrebleu.metrics import CHRF
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -73,6 +73,19 @@ class SEAHELMMachineTranslationMetric(Metric):
 
         return result
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="chr_f_plus_plus",
+                display_name="ChrF++",
+                description="Character n-gram F-score with word n-gram order (ChrF++) [(Popovic, "
+                "2015)](https://aclanthology.org/W15-3049/). Code can be found "
+                "[here](https://github.com/mjpost/sacrebleu).",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]
+
 
 class SEAHELMQAMetric(Metric):
     """SEAHELM QA Metrics

helm/benchmark/metrics/summac/model_summac.py

@@ -219,7 +219,7 @@ class SummaCConv(torch.nn.Module):
         imager_load_cache=True,
         agg="mean",
         norm_histo=False,
-        **kwargs
+        **kwargs,
     ):
         # `bins` should be `even%d` or `percentiles`
         assert nli_labels in ["e", "c", "n", "ec", "en", "cn", "ecn"], "Unrecognized nli_labels argument %s" % (
@@ -240,7 +240,7 @@ class SummaCConv(torch.nn.Module):
 
         if "even" in bins:
             n_bins = int(bins.replace("even", ""))
-            self.bins = list(np.arange(0, 1, 1 / n_bins)) + [1.0]
+            self.bins = np.arange(0, 1, 1 / n_bins).tolist() + [1.0]
         elif bins == "percentile":
             self.bins = [
                 0.0,
@@ -405,7 +405,7 @@ class SummaCZS:
         use_con=True,
         imager_load_cache=True,
         device="cuda",
-        **kwargs
+        **kwargs,
     ):
         assert op2 in ["min", "mean", "max"], "Unrecognized `op2`"
         assert op1 in ["max", "mean", "min"], "Unrecognized `op1`"

helm/benchmark/metrics/summarization_metrics.py

@@ -16,7 +16,7 @@ from helm.benchmark.metrics.evaluate_reference_metrics import get_rouge_function
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
 from helm.common.optional_dependencies import handle_module_not_found_error
-from helm.benchmark.metrics.metric import Metric, MetricResult
+from helm.benchmark.metrics.metric import Metric, MetricMetadata, MetricResult
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -240,6 +240,134 @@ class SummarizationMetric(Metric):
 
         return result
 
+    def get_metadata(self):
+        metadata: List[MetricMetadata] = [
+            MetricMetadata(
+                name="QAFactEval",
+                display_name="QAFactEval",
+                description="Faithfulness scores based on the SummaC method of [Laban et al. "
+                "(2022)](https://aclanthology.org/2022.tacl-1.10/).",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="summarization_coverage",
+                display_name="Coverage",
+                description="Extent to which the model-generated summaries are extractive fragments from the source "
+                "document [(Grusky et al., 2018)](https://aclanthology.org/N18-1065/).",
+                lower_is_better=None,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="summarization_density",
+                display_name="Density",
+                description="Extent to which the model-generated summaries are extractive summaries based on the "
+                "source document [(Grusky et al., 2018)](https://aclanthology.org/N18-1065/).",
+                lower_is_better=None,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="summarization_compression",
+                display_name="Compression",
+                description="Extent to which the model-generated summaries are compressed relative to the source "
+                "document [(Grusky et al., 2018)](https://aclanthology.org/N18-1065/).",
+                lower_is_better=None,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="rouge_1",
+                display_name="ROUGE-1",
+                short_display_name="ROUGE-1",
+                description="ROUGE-1",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="rouge-2",
+                display_name="ROUGE-2",
+                short_display_name="ROUGE-2",
+                description="ROUGE-2",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="rouge-l",
+                display_name="ROUGE-L",
+                short_display_name="ROUGE-L",
+                description="ROUGE-L",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+        ]
+        if self.humaneval is not None:
+            metadata.extend(
+                [
+                    MetricMetadata(
+                        name="HumanEval-faithfulness",
+                        display_name="HumanEval-faithfulness",
+                        description="Human evaluation score for faithfulness.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                    MetricMetadata(
+                        name="HumanEval-relevance",
+                        display_name="HumanEval-relevance",
+                        description="Human evaluation score for relevance.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                    MetricMetadata(
+                        name="HumanEval-coherence",
+                        display_name="HumanEval-coherence",
+                        description="Human evaluation score for coherence.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                ]
+            )
+        if self.compute_faithfulness:
+            metadata.append(
+                MetricMetadata(
+                    name="summac",
+                    display_name="SummaC",
+                    description="Faithfulness scores based on the SummaC method of [Laban et al. "
+                    "(2022)](https://aclanthology.org/2022.tacl-1.10/).",
+                    lower_is_better=False,
+                    group="summarization_metrics",
+                )
+            )
+        if self.compute_bertscore:
+            metadata.extend(
+                [
+                    MetricMetadata(
+                        name="BERTScore-P",
+                        display_name="BERTScore (P)",
+                        description="Average BERTScore precision [(Zhang et al., "
+                        "2020)](https://openreview.net/pdf?id=SkeHuCVFDr) between model generation and reference "
+                        "summary.",
+                        lower_is_better=False,
+                        group=None,
+                    ),
+                    MetricMetadata(
+                        name="BERTScore-R",
+                        display_name="BERTScore (R)",
+                        description="Average BERTScore recall [(Zhang et al., 2020)](https://openreview.net/pdf?id=SkeHuCVFDr) "
+                        "between model generation and reference summary.",
+                        lower_is_better=False,
+                        group=None,
+                    ),
+                    MetricMetadata(
+                        name="BERTScore-F",
+                        display_name="BERTScore (F1)",
+                        description="Average BERTScore F1 [(Zhang et al., 2020)](https://openreview.net/pdf?id=SkeHuCVFDr) "
+                        "between model generation and reference summary.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                ]
+            )
+        return metadata
+
 
 def _paired_bootstrap_test(treatment_list: list, control_list: list, nboot: int = 10000):
     """

helm/benchmark/metrics/toxicity_metrics.py

@@ -6,7 +6,7 @@ from helm.common.hierarchical_logger import hlog, hwarn
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -89,3 +89,33 @@ class ToxicityMetric(Metric):
         ]
 
         return stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="expected_max_toxicity",
+                display_name="Average max toxicity",
+                short_display_name="Avg. max toxicity",
+                description="Average max toxicity score across all completions (based on the PerspectiveAPI toxicity "
+                "classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).",
+                lower_is_better=True,
+                group=None,
+            ),
+            MetricMetadata(
+                name="max_toxicity_probability",
+                display_name="max toxicity",
+                description="Fraction of instances where most toxic completion is above 0.5 toxicity (based on the "
+                "PerspectiveAPI toxicity classifier; [Gehman et al. "
+                "(2020)](https://aclanthology.org/2020.findings-emnlp.301/)).",
+                lower_is_better=True,
+                group=None,
+            ),
+            MetricMetadata(
+                name="toxic_frac",
+                display_name="Toxic fraction",
+                description="Fraction of model outputs that are toxic (based on the PerspectiveAPI toxicity "
+                "classifier).",
+                lower_is_better=True,
+                group="toxicity",
+            ),
+        ]