crfm-helm 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

helm/benchmark/metrics/mimiciv_billing_code_metrics.py

@@ -1,7 +1,7 @@
 from typing import List
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -94,3 +94,34 @@ class MIMICIVBillingCodeMetric(Metric):
             Stat(MetricName("mimiciv_billing_code_recall")).add(recall),
             Stat(MetricName("mimiciv_billing_code_f1")).add(f1),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="mimiciv_billing_code_precision",
+                display_name="Precision for MIMIC Billing Codes",
+                short_display_name="MIMICBillingPre",
+                description="Measures the proportion of correctly predicted ICD codes among all ICD codes predicted by "
+                "the model.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="mimiciv_billing_code_recall",
+                display_name="Recall for MIMIC Billing Codes",
+                short_display_name="MIMICBillingRec",
+                description="Measures the proportion of correctly predicted ICD codes among all ICD codes present in "
+                "the gold standard.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="mimiciv_billing_code_f1",
+                display_name="F1 Score for MIMIC Billing Codes",
+                short_display_name="MIMICBillingF1",
+                description="Measures the harmonic mean of precision and recall for ICD codes, providing a balanced "
+                "evaluation of the model's performance.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/omni_math_metrics.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, List
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -30,3 +30,15 @@ class OmniMATHMetric(Metric):
         return [
             Stat(MetricName("omni_math_accuracy")).add(score),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="omni_math_accuracy",
+                display_name="Omni-MATH Accuracy",
+                short_display_name="Acc",
+                description="Accuracy of the AI output judged by GPT-4.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+        ]

helm/benchmark/metrics/safety_metrics.py

@@ -5,7 +5,7 @@ from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.basic_metrics import compute_request_state_metrics
 from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -77,3 +77,15 @@ class SafetyScoreMetric(Metric):
             raise SafetyScoreMetricException("Could not compute safety score because all annotators failed.")
         stats.append(safety_score_stat)
         return stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="safety_score",
+                display_name="LM Evaluated Safety score",
+                description="LM Evaluated Safety score (mean of Llama3.1 and GPT4o ratings) representing whether the "
+                "output constitutes a successful refusal of unsafe actions (0 is worst, 1 is best).",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/seahelm_metrics.py

@@ -8,7 +8,7 @@ from sacrebleu.metrics import CHRF
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -73,6 +73,19 @@ class SEAHELMMachineTranslationMetric(Metric):
 
         return result
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="chr_f_plus_plus",
+                display_name="ChrF++",
+                description="Character n-gram F-score with word n-gram order (ChrF++) [(Popovic, "
+                "2015)](https://aclanthology.org/W15-3049/). Code can be found "
+                "[here](https://github.com/mjpost/sacrebleu).",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]
+
 
 class SEAHELMQAMetric(Metric):
     """SEAHELM QA Metrics

helm/benchmark/metrics/summac/model_summac.py

@@ -219,7 +219,7 @@ class SummaCConv(torch.nn.Module):
         imager_load_cache=True,
         agg="mean",
         norm_histo=False,
-        **kwargs
+        **kwargs,
     ):
         # `bins` should be `even%d` or `percentiles`
         assert nli_labels in ["e", "c", "n", "ec", "en", "cn", "ecn"], "Unrecognized nli_labels argument %s" % (
@@ -405,7 +405,7 @@ class SummaCZS:
         use_con=True,
         imager_load_cache=True,
         device="cuda",
-        **kwargs
+        **kwargs,
     ):
         assert op2 in ["min", "mean", "max"], "Unrecognized `op2`"
         assert op1 in ["max", "mean", "min"], "Unrecognized `op1`"

helm/benchmark/metrics/summarization_metrics.py

@@ -16,7 +16,7 @@ from helm.benchmark.metrics.evaluate_reference_metrics import get_rouge_function
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
 from helm.common.optional_dependencies import handle_module_not_found_error
-from helm.benchmark.metrics.metric import Metric, MetricResult
+from helm.benchmark.metrics.metric import Metric, MetricMetadata, MetricResult
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -240,6 +240,134 @@ class SummarizationMetric(Metric):
 
         return result
 
+    def get_metadata(self):
+        metadata: List[MetricMetadata] = [
+            MetricMetadata(
+                name="QAFactEval",
+                display_name="QAFactEval",
+                description="Faithfulness scores based on the SummaC method of [Laban et al. "
+                "(2022)](https://aclanthology.org/2022.tacl-1.10/).",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="summarization_coverage",
+                display_name="Coverage",
+                description="Extent to which the model-generated summaries are extractive fragments from the source "
+                "document [(Grusky et al., 2018)](https://aclanthology.org/N18-1065/).",
+                lower_is_better=None,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="summarization_density",
+                display_name="Density",
+                description="Extent to which the model-generated summaries are extractive summaries based on the "
+                "source document [(Grusky et al., 2018)](https://aclanthology.org/N18-1065/).",
+                lower_is_better=None,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="summarization_compression",
+                display_name="Compression",
+                description="Extent to which the model-generated summaries are compressed relative to the source "
+                "document [(Grusky et al., 2018)](https://aclanthology.org/N18-1065/).",
+                lower_is_better=None,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="rouge_1",
+                display_name="ROUGE-1",
+                short_display_name="ROUGE-1",
+                description="ROUGE-1",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="rouge-2",
+                display_name="ROUGE-2",
+                short_display_name="ROUGE-2",
+                description="ROUGE-2",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+            MetricMetadata(
+                name="rouge-l",
+                display_name="ROUGE-L",
+                short_display_name="ROUGE-L",
+                description="ROUGE-L",
+                lower_is_better=False,
+                group="summarization_metrics",
+            ),
+        ]
+        if self.humaneval is not None:
+            metadata.extend(
+                [
+                    MetricMetadata(
+                        name="HumanEval-faithfulness",
+                        display_name="HumanEval-faithfulness",
+                        description="Human evaluation score for faithfulness.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                    MetricMetadata(
+                        name="HumanEval-relevance",
+                        display_name="HumanEval-relevance",
+                        description="Human evaluation score for relevance.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                    MetricMetadata(
+                        name="HumanEval-coherence",
+                        display_name="HumanEval-coherence",
+                        description="Human evaluation score for coherence.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                ]
+            )
+        if self.compute_faithfulness:
+            metadata.append(
+                MetricMetadata(
+                    name="summac",
+                    display_name="SummaC",
+                    description="Faithfulness scores based on the SummaC method of [Laban et al. "
+                    "(2022)](https://aclanthology.org/2022.tacl-1.10/).",
+                    lower_is_better=False,
+                    group="summarization_metrics",
+                )
+            )
+        if self.compute_bertscore:
+            metadata.extend(
+                [
+                    MetricMetadata(
+                        name="BERTScore-P",
+                        display_name="BERTScore (P)",
+                        description="Average BERTScore precision [(Zhang et al., "
+                        "2020)](https://openreview.net/pdf?id=SkeHuCVFDr) between model generation and reference "
+                        "summary.",
+                        lower_is_better=False,
+                        group=None,
+                    ),
+                    MetricMetadata(
+                        name="BERTScore-R",
+                        display_name="BERTScore (R)",
+                        description="Average BERTScore recall [(Zhang et al., 2020)](https://openreview.net/pdf?id=SkeHuCVFDr) "
+                        "between model generation and reference summary.",
+                        lower_is_better=False,
+                        group=None,
+                    ),
+                    MetricMetadata(
+                        name="BERTScore-F",
+                        display_name="BERTScore (F1)",
+                        description="Average BERTScore F1 [(Zhang et al., 2020)](https://openreview.net/pdf?id=SkeHuCVFDr) "
+                        "between model generation and reference summary.",
+                        lower_is_better=False,
+                        group="summarization_metrics",
+                    ),
+                ]
+            )
+        return metadata
+
 
 def _paired_bootstrap_test(treatment_list: list, control_list: list, nboot: int = 10000):
     """

helm/benchmark/metrics/toxicity_metrics.py

@@ -6,7 +6,7 @@ from helm.common.hierarchical_logger import hlog, hwarn
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -89,3 +89,33 @@ class ToxicityMetric(Metric):
         ]
 
         return stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="expected_max_toxicity",
+                display_name="Average max toxicity",
+                short_display_name="Avg. max toxicity",
+                description="Average max toxicity score across all completions (based on the PerspectiveAPI toxicity "
+                "classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).",
+                lower_is_better=True,
+                group=None,
+            ),
+            MetricMetadata(
+                name="max_toxicity_probability",
+                display_name="max toxicity",
+                description="Fraction of instances where most toxic completion is above 0.5 toxicity (based on the "
+                "PerspectiveAPI toxicity classifier; [Gehman et al. "
+                "(2020)](https://aclanthology.org/2020.findings-emnlp.301/)).",
+                lower_is_better=True,
+                group=None,
+            ),
+            MetricMetadata(
+                name="toxic_frac",
+                display_name="Toxic fraction",
+                description="Fraction of model outputs that are toxic (based on the PerspectiveAPI toxicity "
+                "classifier).",
+                lower_is_better=True,
+                group="toxicity",
+            ),
+        ]

helm/benchmark/metrics/ultra_suite_asr_classification_metrics.py

@@ -0,0 +1,52 @@
+from typing import List
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.evaluate_reference_metrics import normalize_text
+from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+)
+from sklearn.metrics import f1_score, accuracy_score
+
+
+class UltraSuiteASRMetric(EvaluateInstancesMetric):
+    """Score metrics for UltraSuite ASR."""
+
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
+        y_pred: List[str] = []
+        y_pred_quasi: List[str] = []
+        y_true: List[str] = []
+        for request_state in request_states:  # one request state per instance
+
+            for reference in request_state.instance.references:
+                if reference.tags == [CORRECT_TAG]:
+                    true_label = reference.output.text
+                    break
+
+            assert request_state.result
+            model_output_text = request_state.result.completions[0].text.strip().lower()
+            assert request_state.instance.extra_data
+            ground_truth_text = request_state.instance.extra_data["transcription"].strip().lower()
+
+            if model_output_text == ground_truth_text:
+                predicted_label = "typically_developing"
+            else:
+                predicted_label = "speech_disorder"
+
+            if normalize_text(predicted_label) == normalize_text(true_label):
+                quasi_label = "typically_developing"
+            else:
+                quasi_label = "speech_disorder"
+
+            y_true.append(true_label)
+            y_pred.append(predicted_label)
+            y_pred_quasi.append(quasi_label)
+
+        return [
+            Stat(MetricName("classification_macro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="macro")),
+            Stat(MetricName("classification_micro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="micro")),
+            Stat(MetricName("exact_match")).add(accuracy_score(y_pred=y_pred, y_true=y_true)),
+            Stat(MetricName("quasi_exact_match")).add(accuracy_score(y_pred=y_pred_quasi, y_true=y_true)),
+        ]

helm/benchmark/metrics/wildbench_metrics.py

@@ -2,7 +2,7 @@ from typing import Any, Dict, List
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -32,3 +32,23 @@ class WildBenchScoreMetric(Metric):
             Stat(MetricName("wildbench_score")).add(score),
             Stat(MetricName("wildbench_score_rescaled")).add(score_rescaled),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="wildbench_score",
+                display_name="WildBench Score",
+                short_display_name="WB Score",
+                description="Score of the AI output judged by GPT-4o.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+            MetricMetadata(
+                name="wildbench_score_rescaled",
+                display_name="WildBench Score",
+                short_display_name="WB Score",
+                description="Score of the AI output judged by GPT-4o, rescaled to be between 0 and 1.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+        ]

helm/benchmark/presentation/run_display.py

@@ -1,6 +1,7 @@
 from collections import OrderedDict, defaultdict
 from dataclasses import dataclass
 import os
+import re
 from typing import Dict, Iterable, List, Optional, Set, Tuple, Any
 
 from helm.benchmark.adaptation.adapter_spec import (
@@ -262,9 +263,18 @@ def write_run_display_json(run_path: str, run_spec: RunSpec, schema: Schema, ski
             if request_state.result is not None and request_state.result.completions
             else ""
         )
-        mapped_output = (
-            request_state.output_mapping.get(predicted_text.strip()) if request_state.output_mapping else None
-        )
+        mapped_output: Optional[str] = None
+        if request_state.output_mapping is not None:
+            output_to_map = predicted_text.strip()
+            if run_spec.adapter_spec.output_mapping_pattern:
+                match = re.search(run_spec.adapter_spec.output_mapping_pattern, output_to_map)
+                if not match:
+                    output_to_map = ""
+                elif match.groups():
+                    output_to_map = match.group(0)
+                else:
+                    output_to_map = match.string
+            mapped_output = request_state.output_mapping.get(output_to_map)
         instance_id_to_instance[(request_state.instance.id, request_state.instance.perturbation)] = (
             request_state.instance
         )

helm/benchmark/presentation/run_entry.py

@@ -14,10 +14,10 @@ class RunEntry:
     description: str
 
     # Priority for this run spec (1 is highest priority, 5 is lowest priority)
-    priority: int
+    priority: Optional[int] = None
 
     # Additional groups to add to the run spec
-    groups: Optional[List[str]]
+    groups: Optional[List[str]] = None
 
 
 @dataclass(frozen=True)

helm/benchmark/presentation/schema.py

@@ -8,6 +8,7 @@ import mako.template
 import yaml
 import importlib_resources as resources
 
+from helm.benchmark.presentation.taxonomy_info import TaxonomyInfo
 from helm.common.general import hlog
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.augmentations.perturbation_description import PERTURBATION_WORST
@@ -131,24 +132,6 @@ THIS_GROUP_ONLY = "this_group_only"
 NO_GROUPS = "no_groups"
 
 
-@dataclass(frozen=True)
-class TaxonomyInfo:
-    # Task (e.g., question answering)
-    task: Optional[str] = None
-
-    # Domain - genre (e.g., Wikipedia)
-    what: Optional[str] = None
-
-    # Domain - when it was written (e.g., 2010s)
-    when: Optional[str] = None
-
-    # Domain - demographics (e.g., web users)
-    who: Optional[str] = None
-
-    # Language (e.g., English)
-    language: Optional[str] = None
-
-
 @dataclass(frozen=True)
 class RunGroup(Field):
     """
@@ -216,16 +199,16 @@ class Schema:
     """Specifies information about what to display on the frontend."""
 
     # Information about each field
-    metrics: List[Field]
+    metrics: List[Field] = field(default_factory=list)
 
     # Information about each perturbation
-    perturbations: List[Field]
+    perturbations: List[Field] = field(default_factory=list)
 
     # Group the metrics
-    metric_groups: List[MetricGroup]
+    metric_groups: List[MetricGroup] = field(default_factory=list)
 
     # Group the scenarios
-    run_groups: List[RunGroup]
+    run_groups: List[RunGroup] = field(default_factory=list)
 
     # Adapter fields (e.g., temperature)
     # Automatically populated from the docstrings in the AdapterSpec class definition.