crfm-helm 0.5.7__py3-none-any.whl → 0.5.9__py3-none-any.whl

helm/benchmark/metrics/alrage_metric.py

@@ -0,0 +1,35 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class ALRAGEMetric(Metric):
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        assert "alrage" in request_state.annotations
+        return [
+            Stat(MetricName("alrage_score")).add(request_state.annotations["alrage"]["score"]),
+        ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="alrage_score",
+                display_name="ALRAGE Score",
+                short_display_name="Score",
+                description="Score of the output judged by GPT-4o.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+        ]

helm/benchmark/metrics/basic_metrics.py

@@ -8,7 +8,7 @@ import numpy as np
 import scipy  # type: ignore
 import calibration as cal  # type: ignore
 from helm.benchmark.adaptation.scenario_state import ScenarioState
-from helm.benchmark.metrics.evaluate_reference_metrics import compute_reference_metrics
+from helm.benchmark.metrics.evaluate_reference_metrics import compute_reference_metrics, get_reference_metrics_metadata
 from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
 from helm.benchmark.metrics.reference_metric import ReferenceMetric
 
@@ -25,7 +25,14 @@ from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
 from helm.benchmark.scenarios.scenario import CORRECT_TAG, Instance
-from helm.benchmark.metrics.metric import Metric, MetricInterface, MetricResult, add_context, get_unique_stat_by_name
+from helm.benchmark.metrics.metric import (
+    Metric,
+    MetricInterface,
+    MetricMetadata,
+    MetricResult,
+    add_context,
+    get_unique_stat_by_name,
+)
 from helm.benchmark.metrics.metric_name import MetricContext, MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat, merge_stat
@@ -104,6 +111,35 @@ def compute_perplexity_metrics(stats: Dict[MetricName, Stat]) -> List[Stat]:
     return derived_stats
 
 
+def _get_perplexity_metrics_metadata() -> List[MetricMetadata]:
+    return [
+        MetricMetadata(
+            name="perplexity",
+            display_name="Perplexity",
+            short_display_name="PPL",
+            description="Perplexity of the output completion (effective branching factor per output token).",
+            lower_is_better=True,
+            group=None,
+        ),
+        MetricMetadata(
+            name="logprob_per_byte",
+            display_name="Log probability / byte",
+            short_display_name="Logprob/byte",
+            description="Predicted output's average log probability normalized by the number of bytes.",
+            lower_is_better=False,
+            group=None,
+        ),
+        MetricMetadata(
+            name="bits_per_byte",
+            display_name="Bits/byte",
+            short_display_name="BPB",
+            description="Average number of bits per byte according to model probabilities.",
+            lower_is_better=True,
+            group=None,
+        ),
+    ]
+
+
 class InstancesPerSplitMetric(MetricInterface):
     """Report the average num_instances in each MetricContext across train_trials."""
 
@@ -133,6 +169,16 @@ class InstancesPerSplitMetric(MetricInterface):
         # There are no per-instance Stats.
         return MetricResult(list(global_stats.values()), [])
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="num_instances",
+                display_name="# eval",
+                description="Number of evaluation instances.",
+                lower_is_better=None,
+            )
+        ]
+
 
 class BasicGenerationMetric(Metric):
     """
@@ -180,6 +226,15 @@ class BasicGenerationMetric(Metric):
         derived_stats.extend(compute_calibration_metrics(per_instance_stats))
         return derived_stats
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        return (
+            get_request_state_metrics_metadata(self.efficiency_metric)
+            + get_reference_metrics_metadata(self.names)
+            + _get_language_modeling_metrics_metadata()
+            + _get_perplexity_metrics_metadata()
+            + _get_calibration_metrics_metadata()
+        )
+
 
 class BasicReferenceMetric(ReferenceMetric):
     """
@@ -295,6 +350,33 @@ class BasicReferenceMetric(ReferenceMetric):
         )
         return stats
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="max_prob",
+                display_name="Max prob",
+                description="Model's average confidence in its prediction (only computed for classification tasks)",
+                lower_is_better=False,
+                group="calibration_detailed",
+            ),
+            MetricMetadata(
+                name="exact_match",
+                display_name="Exact match",
+                short_display_name="EM",
+                description="Fraction of instances that the predicted output matches a correct reference exactly.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+            MetricMetadata(
+                name="predicted_index",
+                display_name="Predicted index",
+                description="Integer index of the reference (0, 1, ...) that was predicted by the model (for "
+                "multiple-choice).",
+                lower_is_better=None,
+                group=None,
+            ),
+        ]
+
 
 def compute_request_state_metrics(
     efficiency_metric: EfficiencyMetric,
@@ -319,6 +401,34 @@ def compute_request_state_metrics(
     return stats
 
 
+def get_request_state_metrics_metadata(
+    efficiency_metric: EfficiencyMetric,
+) -> List[MetricMetadata]:
+    metric_metadata = [
+        MetricMetadata(
+            name="num_references",
+            display_name="# ref",
+            description="Number of references.",
+            lower_is_better=None,
+            group=None,
+        ),
+        MetricMetadata(
+            name="num_train_trials",
+            display_name="# trials",
+            description="Number of trials, where in each trial we choose an independent, random set of training "
+            "instances.",
+            lower_is_better=None,
+            group="general_information",
+        ),
+    ]
+    return (
+        metric_metadata
+        + efficiency_metric.get_metadata()
+        + _get_finish_reason_metrics_metadata()
+        + _get_truncation_metrics_metadata()
+    )
+
+
 def _compute_finish_reason_metrics(
     adapter_spec: AdapterSpec, request_state: RequestState, metric_service: MetricService
 ) -> List[Stat]:
@@ -341,6 +451,40 @@ def _compute_finish_reason_metrics(
     ]
 
 
+def _get_finish_reason_metrics_metadata():
+    return [
+        MetricMetadata(
+            name="finish_reason_endoftext",
+            display_name="finish b/c endoftext",
+            description="Fraction of instances where the the output was terminated because the end of text token "
+            "was generated.",
+            lower_is_better=None,
+            group=None,
+        ),
+        MetricMetadata(
+            name="finish_reason_length",
+            display_name="finish b/c length",
+            description="Fraction of instances where the the output was terminated because of the max tokens limit.",
+            lower_is_better=None,
+            group=None,
+        ),
+        MetricMetadata(
+            name="finish_reason_stop",
+            display_name="finish b/c stop",
+            description="Fraction of instances where the the output was terminated because of the stop sequences.",
+            lower_is_better=None,
+            group=None,
+        ),
+        MetricMetadata(
+            name="finish_reason_unknown",
+            display_name="finish b/c unknown",
+            description="Fraction of instances where the the output was terminated for unknown reasons.",
+            lower_is_better=None,
+            group=None,
+        ),
+    ]
+
+
 def _compute_truncation_metrics(
     adapter_spec: AdapterSpec, request_state: RequestState, metric_service: MetricService
 ) -> List[Stat]:
@@ -354,6 +498,26 @@ def _compute_truncation_metrics(
     ]
 
 
+def _get_truncation_metrics_metadata() -> List[MetricMetadata]:
+    return [
+        MetricMetadata(
+            name="num_train_instances",
+            display_name="# train",
+            description="Number of training instances (e.g., in-context examples).",
+            lower_is_better=None,
+        ),
+        MetricMetadata(
+            name="prompt_truncated",
+            display_name="truncated",
+            description="Fraction of instances where the "
+            "prompt itself was truncated (implies "
+            "that there were no in-context "
+            "examples).",
+            lower_is_better=None,
+        ),
+    ]
+
+
 def compute_language_modeling_metrics(
     adapter_spec: AdapterSpec, request_state: RequestState, metric_service: MetricService
 ) -> List[Stat]:
@@ -387,6 +551,30 @@ def compute_language_modeling_metrics(
     ]
 
 
+def _get_language_modeling_metrics_metadata() -> List[MetricMetadata]:
+    return [
+        MetricMetadata(
+            name="logprob",
+            display_name="Log probability",
+            short_display_name="Logprob",
+            description="Predicted output's average log probability (input's log prob for language modeling).",
+            lower_is_better=False,
+        ),
+        MetricMetadata(
+            name="num_perplexity_tokens",
+            display_name="# tokens",
+            description="Average number of tokens in the predicted output (for language modeling, the input too).",
+            lower_is_better=None,
+        ),
+        MetricMetadata(
+            name="num_bytes",
+            display_name="# bytes",
+            description="Average number of bytes in the predicted output (for language modeling, the input too).",
+            lower_is_better=None,
+        ),
+    ]
+
+
 def _has_non_zero_valued_logprobs(per_instance_stats: Dict[Instance, List[Stat]]) -> bool:
     """Return whether the per-instance stats contain non-zero-valued logprobs.
 
@@ -448,3 +636,80 @@ def compute_calibration_metrics(per_instance_stats: Dict[Instance, List[Stat]])
             stats.append(Stat(MetricName("platt_ece_1_bin")).add(platt_ece_1_bin))
 
     return stats
+
+
+def _get_calibration_metrics_metadata() -> List[MetricMetadata]:
+    return [
+        MetricMetadata(
+            name="ece_10_bin",
+            display_name="10-bin expected calibration error",
+            short_display_name="ECE (10-bin)",
+            description="The average difference between the model's confidence and accuracy, averaged across 10 "
+            "bins where each bin contains an equal number of points (only computed for classification "
+            "tasks). Warning - not reliable for small datasets (e.g., with < 300 examples) because "
+            "each bin will have very few examples.",
+            lower_is_better=True,
+            group="calibration",
+        ),
+        MetricMetadata(
+            name="ece_1_bin",
+            display_name="1-bin expected calibration error",
+            short_display_name="ECE (1-bin)",
+            description="The (absolute value) difference between the model's average confidence and accuracy "
+            "(only computed for classification tasks).",
+            lower_is_better=True,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="selective_acc@10",
+            display_name="Accuracy at 10% coverage",
+            short_display_name="Acc@10%",
+            description="The accuracy for the 10% of predictions that the model is most confident on (only "
+            "computed for classification tasks).",
+            lower_is_better=False,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="selective_cov_acc_area",
+            display_name="Selective coverage-accuracy area",
+            short_display_name="Selective Acc",
+            description="The area under the coverage-accuracy curve, a standard selective classification metric "
+            "(only computed for classification tasks).",
+            lower_is_better=False,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="platt_coef",
+            display_name="Platt Scaling Coefficient",
+            short_display_name="Platt Coef",
+            description="Coefficient of the Platt scaling classifier (can compare this across tasks).",
+            lower_is_better=False,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="platt_intercept",
+            display_name="Platt Scaling Intercept",
+            short_display_name="Platt Intercept",
+            description="Intercept of the Platt scaling classifier (can compare this across tasks).",
+            lower_is_better=False,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="platt_ece_10_bin",
+            display_name="10-bin Expected Calibration Error (after Platt scaling)",
+            short_display_name="Platt-scaled ECE (10-bin)",
+            description="10-bin ECE computed after applying Platt scaling to recalibrate the model's predicted "
+            "probabilities.",
+            lower_is_better=True,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="platt_ece_1_bin",
+            display_name="1-bin expected calibration error (after Platt scaling)",
+            short_display_name="Platt-scaled ECE (1-bin)",
+            description="1-bin ECE computed after applying Platt scaling to recalibrate the model's predicted "
+            "probabilities.",
+            lower_is_better=True,
+            group="calibration_detailed",
+        ),
+    ]

helm/benchmark/metrics/bbq_metrics.py

@@ -1,6 +1,7 @@
 from typing import List
 from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
 
+from helm.benchmark.metrics.metric import MetricMetadata
 from helm.common.request import RequestResult
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.metric_name import MetricName
@@ -145,3 +146,14 @@ class BBQMetric(EvaluateInstancesMetric):
         stats = [acc, amb_bias_stat, disamb_bias_stat]
 
         return stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="bbq_accuracy",
+                display_name="BBQ accuracy",
+                description="BBQ accuracy",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/classification_metrics.py

@@ -6,7 +6,7 @@ from sklearn.preprocessing import MultiLabelBinarizer
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
 from helm.benchmark.metrics.evaluate_reference_metrics import normalize_text
-from helm.benchmark.metrics.metric import MetricName
+from helm.benchmark.metrics.metric import MetricMetadata, MetricName
 from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.scenarios.scenario import Reference
 from helm.common.hierarchical_logger import hwarn
@@ -168,3 +168,21 @@ class MultipleChoiceClassificationMetric(EvaluateInstancesMetric):
             Stat(MetricName("classification_macro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="macro")),
             Stat(MetricName("classification_micro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="micro")),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="classification_macro_f1",
+                display_name="Macro F1",
+                description="Macro F1",
+                lower_is_better=False,
+                group="classification_metrics",
+            ),
+            MetricMetadata(
+                name="classification_micro_f1",
+                display_name="Micro F1",
+                description="Population-level micro-averaged F1 score.",
+                lower_is_better=False,
+                group="classification_metrics",
+            ),
+        ]

helm/benchmark/metrics/conv_fin_qa_calc_metrics.py

@@ -3,7 +3,7 @@ from typing import Any, List
 
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -70,3 +70,14 @@ class ConvFinQACalcMetric(Metric):
         return [
             Stat(MetricName("float_equiv")).add(float_equiv(model_answer, gold_answer)),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="float_equiv",
+                display_name="Float Equivalence",
+                description="Float Equivalence",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/dry_run_metrics.py

@@ -8,7 +8,7 @@ from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.statistic import Stat, merge_stat
 from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
-from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats
+from helm.benchmark.metrics.metric import MetricInterface, MetricMetadata, MetricResult, PerInstanceStats
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.tokens.auto_token_cost_estimator import AutoTokenCostEstimator
@@ -93,3 +93,32 @@ class DryRunMetric(MetricInterface):
         merge_stat(stats, Stat(MetricName("num_requests")).add(len(scenario_state.request_states)))
 
         return MetricResult(list(stats.values()), per_instance_stats)
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="estimated_num_tokens_cost",
+                display_name="cost",
+                short_display_name=None,
+                description="An estimate of the number of tokens (including prompt and output completions) needed to "
+                "perform the request.",
+                lower_is_better=None,
+                group=None,
+            ),
+            MetricMetadata(
+                name="num_completions",
+                display_name="# completions",
+                short_display_name=None,
+                description="Number of completions.",
+                lower_is_better=None,
+                group=None,
+            ),
+            MetricMetadata(
+                name="num_prompt_tokens",
+                display_name="# prompt tokens",
+                short_display_name=None,
+                description="Number of tokens in the prompt.",
+                lower_is_better=None,
+                group="general_information",
+            ),
+        ]

helm/benchmark/metrics/efficiency_metrics.py

@@ -9,6 +9,7 @@ from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
+from helm.benchmark.metrics.metric import MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -162,6 +163,79 @@ class EfficiencyMetric:
             stats.append(Stat(MetricName("inference_idealized_runtime")).add(idealized_runtime))
         return stats
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="num_prompt_tokens",
+                display_name="# prompt tokens",
+                short_display_name=None,
+                description="Number of tokens in the prompt.",
+                lower_is_better=None,
+                group="general_information",
+            ),
+            MetricMetadata(
+                name="num_completion_tokens",
+                display_name="# completion tokens",
+                description="Actual number of completion tokens (over all completions).",
+                lower_is_better=None,
+            ),
+            MetricMetadata(
+                name="num_output_tokens",
+                display_name="# output tokens",
+                description="Actual number of output tokens.",
+                lower_is_better=None,
+            ),
+            MetricMetadata(
+                name="training_co2_cost",
+                display_name="Estimated training emissions (kg CO2)",
+                short_display_name="Training emissions (kg CO2)",
+                description="Estimate of the CO2 emissions from training the model.",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="training_energy_cost",
+                display_name="Estimated training energy cost (MWh)",
+                short_display_name="Training energy (MWh)",
+                description="Estimate of the amount of energy used to train the model.",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="inference_runtime",
+                display_name="Observed inference runtime (s)",
+                short_display_name="Observed inference time (s)",
+                description="Average observed time to process a request to the model (via an API, and thus depends on "
+                "particular deployment).",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="batch_size",
+                display_name="Batch size",
+                description="For batch jobs, how many requests are in a batch.",
+                lower_is_better=None,
+            ),
+            MetricMetadata(
+                name="inference_denoised_runtime",
+                display_name="Denoised inference runtime (s)",
+                short_display_name="Denoised inference time (s)",
+                description="Average time to process a request to the model minus performance contention by using "
+                "profiled runtimes from multiple trials of SyntheticEfficiencyScenario.",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+            MetricMetadata(
+                name="inference_idealized_runtime",
+                display_name="Idealized inference runtime (s)",
+                short_display_name="Idealized inference time (s)",
+                description="Average time to process a request to the model based solely on the model architecture "
+                "(using Megatron-LM).",
+                lower_is_better=True,
+                group="efficiency_detailed",
+            ),
+        ]
+
 
 def _compute_estimated_time_from_prompt_size_and_num_output_tokens(
     request_state: RequestState,

helm/benchmark/metrics/ehr_sql_metrics.py

@@ -1,7 +1,7 @@
 from typing import List
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
 from helm.benchmark.metrics.metric_name import MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
@@ -101,3 +101,59 @@ class EhrSqlMetric(Metric):
             Stat(MetricName("ehr_sql_total_predicted_answerable")).add(int(is_predicted_answerable)),
             Stat(MetricName("ehr_sql_total_ground_truth_answerable")).add(int(is_answerable)),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="ehr_sql_execution_accuracy",
+                display_name="Execution accuracy for Generated Query",
+                short_display_name="EHRSQLExeAcc",
+                description="Measures the proportion of correctly predicted answerable questions among all questions "
+                "predicted to be answerable.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_query_validity",
+                display_name="Validity of Generated Query",
+                short_display_name="EHRSQLQueryValid",
+                description="Measures the proportion of correctly predicted answerable questions among all answerable "
+                "questions in the dataset.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_precision_answerable",
+                display_name="Precision for Answerable Questions",
+                short_display_name="EHRSQLPreAns",
+                description="Measures the proportion of correctly predicted answerable questions among all questions "
+                "predicted to be answerable.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_recall_answerable",
+                display_name="Recall for Answerable Questions",
+                short_display_name="EHRSQLReAns",
+                description="Measures the proportion of correctly predicted answerable questions among all answerable "
+                "questions in the dataset.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_total_predicted_answerable",
+                display_name="Total Predicted Answerable",
+                short_display_name="Total Pred Ans",
+                description="Total number of questions predicted to be answerable by the model.",
+                lower_is_better=False,
+                group=None,
+            ),
+            MetricMetadata(
+                name="ehr_sql_total_ground_truth_answerable",
+                display_name="Total Ground Truth Answerable",
+                short_display_name="Total GT Ans",
+                description="Total number of answerable questions in the ground truth.",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]