crfm-helm 0.5.6__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/metrics/alrage_metric.py

@@ -0,0 +1,35 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class ALRAGEMetric(Metric):
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        assert "alrage" in request_state.annotations
+        return [
+            Stat(MetricName("alrage_score")).add(request_state.annotations["alrage"]["score"]),
+        ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="alrage_score",
+                display_name="ALRAGE Score",
+                short_display_name="Score",
+                description="Score of the output judged by GPT-4o.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+        ]

helm/benchmark/metrics/basic_metrics.py

@@ -8,7 +8,7 @@ import numpy as np
 import scipy  # type: ignore
 import calibration as cal  # type: ignore
 from helm.benchmark.adaptation.scenario_state import ScenarioState
-from helm.benchmark.metrics.evaluate_reference_metrics import compute_reference_metrics
+from helm.benchmark.metrics.evaluate_reference_metrics import compute_reference_metrics, get_reference_metrics_metadata
 from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
 from helm.benchmark.metrics.reference_metric import ReferenceMetric
 
@@ -25,7 +25,14 @@ from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
 from helm.benchmark.scenarios.scenario import CORRECT_TAG, Instance
-from helm.benchmark.metrics.metric import Metric, MetricInterface, MetricResult, add_context, get_unique_stat_by_name
+from helm.benchmark.metrics.metric import (
+    Metric,
+    MetricInterface,
+    MetricMetadata,
+    MetricResult,
+    add_context,
+    get_unique_stat_by_name,
+)
 from helm.benchmark.metrics.metric_name import MetricContext, MetricName
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat, merge_stat
@@ -104,6 +111,35 @@ def compute_perplexity_metrics(stats: Dict[MetricName, Stat]) -> List[Stat]:
     return derived_stats
 
 
+def _get_perplexity_metrics_metadata() -> List[MetricMetadata]:
+    return [
+        MetricMetadata(
+            name="perplexity",
+            display_name="Perplexity",
+            short_display_name="PPL",
+            description="Perplexity of the output completion (effective branching factor per output token).",
+            lower_is_better=True,
+            group=None,
+        ),
+        MetricMetadata(
+            name="logprob_per_byte",
+            display_name="Log probability / byte",
+            short_display_name="Logprob/byte",
+            description="Predicted output's average log probability normalized by the number of bytes.",
+            lower_is_better=False,
+            group=None,
+        ),
+        MetricMetadata(
+            name="bits_per_byte",
+            display_name="Bits/byte",
+            short_display_name="BPB",
+            description="Average number of bits per byte according to model probabilities.",
+            lower_is_better=True,
+            group=None,
+        ),
+    ]
+
+
 class InstancesPerSplitMetric(MetricInterface):
     """Report the average num_instances in each MetricContext across train_trials."""
 
@@ -133,6 +169,16 @@ class InstancesPerSplitMetric(MetricInterface):
         # There are no per-instance Stats.
         return MetricResult(list(global_stats.values()), [])
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="num_instances",
+                display_name="# eval",
+                description="Number of evaluation instances.",
+                lower_is_better=None,
+            )
+        ]
+
 
 class BasicGenerationMetric(Metric):
     """
@@ -180,6 +226,15 @@ class BasicGenerationMetric(Metric):
         derived_stats.extend(compute_calibration_metrics(per_instance_stats))
         return derived_stats
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        return (
+            get_request_state_metrics_metadata(self.efficiency_metric)
+            + get_reference_metrics_metadata(self.names)
+            + _get_language_modeling_metrics_metadata()
+            + _get_perplexity_metrics_metadata()
+            + _get_calibration_metrics_metadata()
+        )
+
 
 class BasicReferenceMetric(ReferenceMetric):
     """
@@ -295,6 +350,33 @@ class BasicReferenceMetric(ReferenceMetric):
         )
         return stats
 
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="max_prob",
+                display_name="Max prob",
+                description="Model's average confidence in its prediction (only computed for classification tasks)",
+                lower_is_better=False,
+                group="calibration_detailed",
+            ),
+            MetricMetadata(
+                name="exact_match",
+                display_name="Exact match",
+                short_display_name="EM",
+                description="Fraction of instances that the predicted output matches a correct reference exactly.",
+                lower_is_better=False,
+                group="accuracy",
+            ),
+            MetricMetadata(
+                name="predicted_index",
+                display_name="Predicted index",
+                description="Integer index of the reference (0, 1, ...) that was predicted by the model (for "
+                "multiple-choice).",
+                lower_is_better=None,
+                group=None,
+            ),
+        ]
+
 
 def compute_request_state_metrics(
     efficiency_metric: EfficiencyMetric,
@@ -319,6 +401,34 @@ def compute_request_state_metrics(
     return stats
 
 
+def get_request_state_metrics_metadata(
+    efficiency_metric: EfficiencyMetric,
+) -> List[MetricMetadata]:
+    metric_metadata = [
+        MetricMetadata(
+            name="num_references",
+            display_name="# ref",
+            description="Number of references.",
+            lower_is_better=None,
+            group=None,
+        ),
+        MetricMetadata(
+            name="num_train_trials",
+            display_name="# trials",
+            description="Number of trials, where in each trial we choose an independent, random set of training "
+            "instances.",
+            lower_is_better=None,
+            group="general_information",
+        ),
+    ]
+    return (
+        metric_metadata
+        + efficiency_metric.get_metadata()
+        + _get_finish_reason_metrics_metadata()
+        + _get_truncation_metrics_metadata()
+    )
+
+
 def _compute_finish_reason_metrics(
     adapter_spec: AdapterSpec, request_state: RequestState, metric_service: MetricService
 ) -> List[Stat]:
@@ -341,6 +451,40 @@ def _compute_finish_reason_metrics(
     ]
 
 
+def _get_finish_reason_metrics_metadata():
+    return [
+        MetricMetadata(
+            name="finish_reason_endoftext",
+            display_name="finish b/c endoftext",
+            description="Fraction of instances where the the output was terminated because the end of text token "
+            "was generated.",
+            lower_is_better=None,
+            group=None,
+        ),
+        MetricMetadata(
+            name="finish_reason_length",
+            display_name="finish b/c length",
+            description="Fraction of instances where the the output was terminated because of the max tokens limit.",
+            lower_is_better=None,
+            group=None,
+        ),
+        MetricMetadata(
+            name="finish_reason_stop",
+            display_name="finish b/c stop",
+            description="Fraction of instances where the the output was terminated because of the stop sequences.",
+            lower_is_better=None,
+            group=None,
+        ),
+        MetricMetadata(
+            name="finish_reason_unknown",
+            display_name="finish b/c unknown",
+            description="Fraction of instances where the the output was terminated for unknown reasons.",
+            lower_is_better=None,
+            group=None,
+        ),
+    ]
+
+
 def _compute_truncation_metrics(
     adapter_spec: AdapterSpec, request_state: RequestState, metric_service: MetricService
 ) -> List[Stat]:
@@ -354,6 +498,26 @@ def _compute_truncation_metrics(
     ]
 
 
+def _get_truncation_metrics_metadata() -> List[MetricMetadata]:
+    return [
+        MetricMetadata(
+            name="num_train_instances",
+            display_name="# train",
+            description="Number of training instances (e.g., in-context examples).",
+            lower_is_better=None,
+        ),
+        MetricMetadata(
+            name="prompt_truncated",
+            display_name="truncated",
+            description="Fraction of instances where the "
+            "prompt itself was truncated (implies "
+            "that there were no in-context "
+            "examples).",
+            lower_is_better=None,
+        ),
+    ]
+
+
 def compute_language_modeling_metrics(
     adapter_spec: AdapterSpec, request_state: RequestState, metric_service: MetricService
 ) -> List[Stat]:
@@ -387,6 +551,30 @@ def compute_language_modeling_metrics(
     ]
 
 
+def _get_language_modeling_metrics_metadata() -> List[MetricMetadata]:
+    return [
+        MetricMetadata(
+            name="logprob",
+            display_name="Log probability",
+            short_display_name="Logprob",
+            description="Predicted output's average log probability (input's log prob for language modeling).",
+            lower_is_better=False,
+        ),
+        MetricMetadata(
+            name="num_perplexity_tokens",
+            display_name="# tokens",
+            description="Average number of tokens in the predicted output (for language modeling, the input too).",
+            lower_is_better=None,
+        ),
+        MetricMetadata(
+            name="num_bytes",
+            display_name="# bytes",
+            description="Average number of bytes in the predicted output (for language modeling, the input too).",
+            lower_is_better=None,
+        ),
+    ]
+
+
 def _has_non_zero_valued_logprobs(per_instance_stats: Dict[Instance, List[Stat]]) -> bool:
     """Return whether the per-instance stats contain non-zero-valued logprobs.
 
@@ -448,3 +636,80 @@ def compute_calibration_metrics(per_instance_stats: Dict[Instance, List[Stat]])
             stats.append(Stat(MetricName("platt_ece_1_bin")).add(platt_ece_1_bin))
 
     return stats
+
+
+def _get_calibration_metrics_metadata() -> List[MetricMetadata]:
+    return [
+        MetricMetadata(
+            name="ece_10_bin",
+            display_name="10-bin expected calibration error",
+            short_display_name="ECE (10-bin)",
+            description="The average difference between the model's confidence and accuracy, averaged across 10 "
+            "bins where each bin contains an equal number of points (only computed for classification "
+            "tasks). Warning - not reliable for small datasets (e.g., with < 300 examples) because "
+            "each bin will have very few examples.",
+            lower_is_better=True,
+            group="calibration",
+        ),
+        MetricMetadata(
+            name="ece_1_bin",
+            display_name="1-bin expected calibration error",
+            short_display_name="ECE (1-bin)",
+            description="The (absolute value) difference between the model's average confidence and accuracy "
+            "(only computed for classification tasks).",
+            lower_is_better=True,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="selective_acc@10",
+            display_name="Accuracy at 10% coverage",
+            short_display_name="Acc@10%",
+            description="The accuracy for the 10% of predictions that the model is most confident on (only "
+            "computed for classification tasks).",
+            lower_is_better=False,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="selective_cov_acc_area",
+            display_name="Selective coverage-accuracy area",
+            short_display_name="Selective Acc",
+            description="The area under the coverage-accuracy curve, a standard selective classification metric "
+            "(only computed for classification tasks).",
+            lower_is_better=False,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="platt_coef",
+            display_name="Platt Scaling Coefficient",
+            short_display_name="Platt Coef",
+            description="Coefficient of the Platt scaling classifier (can compare this across tasks).",
+            lower_is_better=False,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="platt_intercept",
+            display_name="Platt Scaling Intercept",
+            short_display_name="Platt Intercept",
+            description="Intercept of the Platt scaling classifier (can compare this across tasks).",
+            lower_is_better=False,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="platt_ece_10_bin",
+            display_name="10-bin Expected Calibration Error (after Platt scaling)",
+            short_display_name="Platt-scaled ECE (10-bin)",
+            description="10-bin ECE computed after applying Platt scaling to recalibrate the model's predicted "
+            "probabilities.",
+            lower_is_better=True,
+            group="calibration_detailed",
+        ),
+        MetricMetadata(
+            name="platt_ece_1_bin",
+            display_name="1-bin expected calibration error (after Platt scaling)",
+            short_display_name="Platt-scaled ECE (1-bin)",
+            description="1-bin ECE computed after applying Platt scaling to recalibrate the model's predicted "
+            "probabilities.",
+            lower_is_better=True,
+            group="calibration_detailed",
+        ),
+    ]

helm/benchmark/metrics/bbq_metrics.py

@@ -1,6 +1,7 @@
 from typing import List
 from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
 
+from helm.benchmark.metrics.metric import MetricMetadata
 from helm.common.request import RequestResult
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.metric_name import MetricName
@@ -145,3 +146,14 @@ class BBQMetric(EvaluateInstancesMetric):
         stats = [acc, amb_bias_stat, disamb_bias_stat]
 
         return stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="bbq_accuracy",
+                display_name="BBQ accuracy",
+                description="BBQ accuracy",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/classification_metrics.py

@@ -6,7 +6,7 @@ from sklearn.preprocessing import MultiLabelBinarizer
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
 from helm.benchmark.metrics.evaluate_reference_metrics import normalize_text
-from helm.benchmark.metrics.metric import MetricName
+from helm.benchmark.metrics.metric import MetricMetadata, MetricName
 from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.scenarios.scenario import Reference
 from helm.common.hierarchical_logger import hwarn
@@ -168,3 +168,21 @@ class MultipleChoiceClassificationMetric(EvaluateInstancesMetric):
             Stat(MetricName("classification_macro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="macro")),
             Stat(MetricName("classification_micro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="micro")),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="classification_macro_f1",
+                display_name="Macro F1",
+                description="Macro F1",
+                lower_is_better=False,
+                group="classification_metrics",
+            ),
+            MetricMetadata(
+                name="classification_micro_f1",
+                display_name="Micro F1",
+                description="Population-level micro-averaged F1 score.",
+                lower_is_better=False,
+                group="classification_metrics",
+            ),
+        ]

helm/benchmark/metrics/codeinsights_code_efficiency_metrics.py

@@ -0,0 +1,186 @@
+from typing import List, Tuple, Dict, Any
+import time
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.codeinsights_correct_code_metrics import (
+    CodeInsightsFunctionalCorrectnessMetric,
+    CPPEvaluator,
+)
+
+
+class CodeInsightsCodeEfficiencyMetric(CodeInsightsFunctionalCorrectnessMetric):
+    """
+    Comprehensive metric combining functional correctness and runtime efficiency evaluation.
+
+    This metric first evaluates functional correctness and then measures runtime efficiency
+    alignment between LLM-generated code and student reference code when both are correct.
+    """
+
+    def __init__(
+        self,
+        num_runtime_runs: int = 5,
+        timeout_seconds: int = 10,
+    ):
+        """
+        Initializes the CodeInsightsFunctionalCorrectnessMetric.
+
+        Args:
+            timeout (int): Timeout for each test case execution.
+        """
+        super().__init__()
+        self.num_runtime_runs = num_runtime_runs
+        self.timeout_seconds = timeout_seconds
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """
+        Evaluate LLM-generated code by running unit tests and computing pass rate.
+
+        Returns:
+            List of Stat objects containing the functional correctness score
+        """
+        print("\n=== FUNCTIONAL CORRECTNESS METRIC DEBUG ===")
+        print(f"Instance ID: {getattr(request_state.instance, 'id', 'UNKNOWN')}")
+
+        # Get the generated code from the request state
+        if not request_state.result or not request_state.result.completions:
+            print("ERROR: No output generated")
+            return self._create_failure_stats("No output generated")
+
+        generated_code = request_state.result.completions[0].text.strip()
+        generated_code = self._extract_student_code(generated_code)
+        print(f"Generated code length: {len(generated_code)}")
+        print(f"Generated code preview: {generated_code[:200]}...")
+
+        # Get the student code from the instance references
+        student_code = request_state.instance.references[0].output.text.strip()
+        print(f"Student code length: {len(student_code)}")
+
+        # Get test cases from instance extra_data
+        if not hasattr(request_state.instance, "extra_data") or not request_state.instance.extra_data:
+            print("ERROR: No extra_data available")
+            print(f"Instance attributes: {dir(request_state.instance)}")
+            return self._create_failure_stats("No test data available")
+
+        extra_data = request_state.instance.extra_data
+        print(f"Extra data keys: {list(extra_data.keys())}")
+
+        test_cases = extra_data.get("test_cases", [])
+        question_template = extra_data.get("question_template", "")
+        question_name = extra_data.get("question_name", "UNKNOWN")
+
+        print(f"Question name: {question_name}")
+        print(f"Number of test cases: {len(test_cases)}")
+        print(f"Template length: {len(question_template)}")
+
+        if not test_cases:
+            print("ERROR: No test cases available")
+            return self._create_failure_stats("No test cases available")
+
+        print(f"First test case preview: {test_cases[0] if test_cases else 'NONE'}")
+
+        # Run unit tests and calculate pass rate
+        evaluator = CPPEvaluator(
+            question_template,
+            test_cases,
+            timeout=self.timeout_seconds,
+            max_workers=1,
+        )
+
+        llm_output, llm_avg_runtime = self._timed_run(evaluator, generated_code, self.num_runtime_runs)
+        stu_output, stu_avg_runtime = self._timed_run(evaluator, student_code, self.num_runtime_runs)
+
+        # Compute functional correctness score
+        if not llm_output or "score" not in llm_output:
+            stats = [Stat(MetricName("functional_correctness")).add(0.0)]
+        else:
+            stats = [Stat(MetricName("functional_correctness")).add(llm_output["score"])]
+
+        # Calculate runtime metrics if we have data for both solutions
+        if llm_avg_runtime > 0 and stu_avg_runtime > 0:
+            # Runtime ratio (LLM / Student) - values > 1 mean LLM is slower
+            runtime_ratio = llm_avg_runtime / stu_avg_runtime if stu_avg_runtime > 0 else float("inf")
+
+            # Efficiency alignment score (closer to 1.0 is better alignment)
+            # Use reciprocal if LLM is faster to normalize the scale
+            if runtime_ratio > 1:
+                efficiency_alignment = 1.0 / runtime_ratio
+            else:
+                efficiency_alignment = runtime_ratio
+
+            print(f"Runtime ratio (LLM/Student): {runtime_ratio:.4f}")
+            print(f"Efficiency alignment score: {efficiency_alignment:.4f}")
+
+            stats.extend(
+                [
+                    Stat(MetricName("runtime_efficiency_ratio")).add(runtime_ratio),
+                    Stat(MetricName("efficiency_alignment_score")).add(efficiency_alignment),
+                ]
+            )
+
+        # Handle cases where only one solution has runtime data
+        elif llm_avg_runtime > 0 and stu_avg_runtime <= 0:
+            print("Only LLM runtime available - student solution failed to run")
+            stats.extend(
+                [
+                    Stat(MetricName("runtime_efficiency_ratio")).add(float("inf")),  # LLM runs, student doesn't
+                    Stat(MetricName("efficiency_alignment_score")).add(0.0),  # No alignment possible
+                ]
+            )
+
+        elif llm_avg_runtime <= 0 and stu_avg_runtime > 0:
+            print("Only student runtime available - LLM solution failed to run")
+            stats.extend(
+                [
+                    Stat(MetricName("runtime_efficiency_ratio")).add(0.0),  # Student runs, LLM doesn't
+                    Stat(MetricName("efficiency_alignment_score")).add(0.0),  # No alignment possible
+                ]
+            )
+
+        else:
+            # Neither solution has runtime data
+            print("Runtime measurement failed for both solutions")
+            stats.extend(
+                [
+                    Stat(MetricName("runtime_efficiency_ratio")).add(0.0),
+                    Stat(MetricName("efficiency_alignment_score")).add(0.0),
+                ]
+            )
+
+        return stats
+
+    def _timed_run(self, evaluator: CPPEvaluator, code: str, num_runtime_runs: int = 1) -> Tuple[Dict[str, Any], float]:
+        list_runtimes: List[float] = []
+        last_output: Dict[str, Any] = {}
+
+        for _ in range(num_runtime_runs):
+            start_time = time.perf_counter()
+            output = evaluator.evaluate(code)
+            passed = sum(output.get("testcases", []))
+
+            if passed > 0:
+                elapsed = time.perf_counter() - start_time
+                list_runtimes.append(elapsed / passed)
+                last_output = output
+            # if passed == 0, we simply skip recording this run
+
+        avg_runtime = sum(list_runtimes) / len(list_runtimes) if list_runtimes else 0.0
+        return last_output, avg_runtime
+
+    def _create_failure_stats(self, error_message: str) -> List[Stat]:
+        """Create default statistics for failure cases."""
+        print(f"RUNTIME EFFICIENCY METRIC FAILURE: {error_message}")
+        return [
+            Stat(MetricName("functional_correctness")).add(0.0),
+            Stat(MetricName("runtime_efficiency_ratio")).add(0.0),
+            Stat(MetricName("efficiency_alignment_score")).add(0.0),
+        ]