crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/metrics/bbq_metrics.py

@@ -1,10 +1,11 @@
 from typing import List
+from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
 
+from helm.benchmark.metrics.metric import MetricMetadata
 from helm.common.request import RequestResult
 from helm.benchmark.adaptation.request_state import RequestState
-from .metric import Metric
-from .metric_name import MetricName
-from .statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
 
 AMBIGUOUS_TAG = "ambiguous"
 NON_AMBIGUOUS_TAG = "non-ambiguous"
@@ -13,7 +14,7 @@ NON_NEGATIVE_TAG = "non-negative"
 ERROR_SCORE = 0.0  # fallback value after experiencing a divide-by-0 error when computing BBQMetric
 
 
-class BBQMetric(Metric):
+class BBQMetric(EvaluateInstancesMetric):
     """
     Defines metrics for the BBQ dataset. Specifically, we are interested in the following metrics:
 
@@ -49,8 +50,7 @@ class BBQMetric(Metric):
     For more details, see the equation on page 6 of https://arxiv.org/pdf/2110.08193.pdf
     """
 
-    def evaluate_instances(self, request_states: List[RequestState]) -> List[Stat]:
-
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
         amb_non_unknown = 0  # keep track of the number of non-unknowns
         disamb_non_unknown = 0  # keep track of the number of non-unknowns
 
@@ -146,3 +146,14 @@ class BBQMetric(Metric):
         stats = [acc, amb_bias_stat, disamb_bias_stat]
 
         return stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="bbq_accuracy",
+                display_name="BBQ accuracy",
+                description="BBQ accuracy",
+                lower_is_better=False,
+                group=None,
+            ),
+        ]

helm/benchmark/metrics/bias_metrics.py

@@ -4,16 +4,25 @@ from typing import Dict, List, Optional, Tuple
 
 from nltk.tokenize import word_tokenize
 import numpy as np
+from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
 
-from helm.common.request import RequestResult, Sequence
+from helm.common.request import RequestResult, GeneratedOutput
 from helm.benchmark.adaptation.request_state import RequestState
-from .statistic import Stat
-from .metric import Metric
-from .metric_name import MetricName
-from .bias_word_lists import GENDER_TO_WORD_LISTS, RACE_TO_NAME_LISTS, ADJECTIVE_LIST, PROFESSION_LIST
+from helm.benchmark.metrics.nltk_helper import install_nltk_resources
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.bias_word_lists import (
+    GENDER_TO_WORD_LISTS,
+    RACE_TO_NAME_LISTS,
+    ADJECTIVE_LIST,
+    PROFESSION_LIST,
+)
 
 
-class BiasMetric(Metric):
+install_nltk_resources()
+
+
+class BiasMetric(EvaluateInstancesMetric):
     """Compute metrics to evaluate social bias.
 
     We compute demographic representation and mean stereotypical association bias in model generated text using word
@@ -155,7 +164,7 @@ class BiasMetric(Metric):
         pair_to_count: Dict[Tuple[str, str], int] = defaultdict(int)
         for text in texts:
             tokens = word_tokenize(text.lower())
-            for (target_word, group) in itertools.product(target_words, demographic_groups):
+            for target_word, group in itertools.product(target_words, demographic_groups):
                 group_words = self.demographic_group_to_words[group]
                 num_group_tokens = sum([tokens.count(w) for w in group_words])  # e.g. number of time asian names occur
                 num_target_tokens = tokens.count(target_word)  # e.g. number of tokens that are in the adj list
@@ -214,12 +223,12 @@ class BiasMetric(Metric):
 
         return tv_distance
 
-    def evaluate_instances(self, request_states: List[RequestState]) -> List[Stat]:
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
         """Compute the bias score on the request_states."""
 
         # Get completion texts from the request_results
         request_results: List[RequestResult] = [rs.result for rs in request_states if rs.result]
-        completions: List[Sequence] = [c for rr in request_results for c in rr.completions if rr.completions]
+        completions: List[GeneratedOutput] = [c for rr in request_results for c in rr.completions if rr.completions]
         completion_texts: List[str] = [c.text for c in completions if c.text]
 
         # Compute the bias score

helm/benchmark/metrics/bias_word_lists.py

@@ -1,4 +1,4 @@
-""" Bias words utilised to compute the bias metrics.
+"""Bias words utilised to compute the bias metrics.
 
 This file includes word lists for the following 4 categories:
     - Race (Asian, Hispanic, and White): Most common names for each race (Garg et al. 2018)

helm/benchmark/metrics/bigcodebench_metrics.py

@@ -0,0 +1,25 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class BigCodeBenchMetric(Metric):
+    """Score metrics for BigCodeBench."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        score = request_state.annotations["bigcodebench"]["pass_at_one"]
+        return [
+            Stat(MetricName("bigcodebench_p@1")).add(score),
+        ]

helm/benchmark/metrics/bird_sql_metrics.py

@@ -0,0 +1,28 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class BirdSQLMetric(Metric):
+    """Score metrics for Bird-SQL."""
+
+    ANNOTATOR_NAME = "bird_sql"
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        if not request_state.annotations:
+            raise Exception("Request state did not have annotations.")
+        predicted_result = request_state.annotations[self.ANNOTATOR_NAME]["predicted_result"]
+        ground_truth_result = request_state.annotations[self.ANNOTATOR_NAME]["ground_truth_result"]
+        execution_accuracy = int(set(predicted_result) == set(ground_truth_result))
+        return [Stat(MetricName("execution_accuracy")).add(execution_accuracy)]

helm/benchmark/metrics/classification_metrics.py

@@ -1,17 +1,23 @@
 from typing import List, Optional
 
-from sklearn.metrics import f1_score
+from sklearn.metrics import f1_score, precision_score, recall_score
 from sklearn.preprocessing import MultiLabelBinarizer
 
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.basic_metrics import normalize_text
-from helm.benchmark.metrics.metric import Metric, MetricName
+from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
+from helm.benchmark.metrics.evaluate_reference_metrics import normalize_text
+from helm.benchmark.metrics.metric import MetricMetadata, MetricName
 from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.scenarios.scenario import Reference
-from helm.common.request import Sequence
+from helm.common.hierarchical_logger import hwarn
+from helm.common.request import GeneratedOutput
 
 
-class ClassificationMetric(Metric):
+def _normalize_label_text(text: str) -> str:
+    return normalize_text(text, should_remove_articles=False)
+
+
+class ClassificationMetric(EvaluateInstancesMetric):
     """Defines metrics for multi-class classification using the generation adapter.
 
     Currently provides `classification_macro_f1` and `classification_micro_f1`.
@@ -22,8 +28,12 @@ class ClassificationMetric(Metric):
     reference. The predicted class for each instance is the normalized text of the generation.
 
     Note:
-    - The set of classes is derived from the correct references from all the instances.
-      This means that classes may be omitted if they are never used as a correct reference.
+    - It is highly recommended to specify the set of classes should be specified using the
+      `labels` parameter. Otherwise, the set of classes is derived from the correct references
+      from all the instances. This means that classes may be incorrectly omitted if they are never
+      used as a correct reference.
+    - The `averages` parameter is a list of averaging methods to be used.
+      It has the same meaning `average` as in scikit-learn.
     - Generations that are not in any of the known classes are counted as a
       negative prediction for every class.
     - Perturbed classes are considered different classes from unperturbed
@@ -31,13 +41,50 @@ class ClassificationMetric(Metric):
     - Currently, multi-label classification is not supported.
     """
 
-    def __init__(self, delimiter: Optional[str] = None):
+    AVERAGE_OPTIONS = ["micro", "macro", "weighted", None]
+    SCORE_OPTIONS = ["f1", "precision", "recall"]
+
+    def __init__(
+        self,
+        averages: Optional[List[Optional[str]]] = None,
+        labels: Optional[List[str]] = None,
+        scores: Optional[List[str]] = None,
+        delimiter: Optional[str] = None,
+    ) -> None:
+        """Creates metrics for multi-class classification.
+
+        :param delimiter: For multi-label classification, the string delimiter between classes in the model's output.
+        :param average: The list of scores to compute (e.g. "f1", "precision", "recall").
+          Defaults to ["f1"].
+        :param average: The averaging methods (e.g. "micro", "macro", "weighted") to be used.
+          It has the same meaning `average` as in scikit-learn.
+          Defaults to ["macro", "micro"].
+        :param labels: The set of labels.
+        :return: A list of `Stat` objects.
+        """
+        self.averages = averages or ["macro", "micro"]
+        for average in self.averages:
+            if average not in ClassificationMetric.AVERAGE_OPTIONS:
+                raise ValueError(
+                    f"Each value in `averages` must be set to one of {ClassificationMetric.AVERAGE_OPTIONS}."
+                )
+        self.scores = scores or ["f1"]
+        for score_name in self.scores:
+            if score_name not in ClassificationMetric.SCORE_OPTIONS:
+                raise ValueError(f"Each value in `scores` must be set to one of {ClassificationMetric.SCORE_OPTIONS}.")
         self.delimiter = delimiter
+        self.labels = labels
+        if not self.labels:
+            hwarn(
+                "`labels` were not set on `ClassificationMetric`, "
+                "so they will be inferred from target references. "
+                "It is recommend to explicitly set `labels` on `ClassificationMetric`."
+            )
 
     def is_multi_label(self) -> bool:
         return bool(self.delimiter)
 
-    def evaluate_instances(self, request_states: List[RequestState]) -> List[Stat]:
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
         y_pred: List[List[str]] = []
         y_true: List[List[str]] = []
         for request_state in request_states:  # one request state per instance
@@ -56,29 +103,47 @@ class ClassificationMetric(Metric):
             references = request_state.instance.all_correct_references
             if not self.is_multi_label():
                 assert len(references) == 1
-            correct_ref_texts = [normalize_text(ref.output.text) for ref in references if ref.output.text]
+            correct_ref_texts = [_normalize_label_text(ref.output.text) for ref in references if ref.output.text]
             y_true.append(correct_ref_texts)
 
             input_text = request_state.result.completions[0].text
             predictions = input_text.split(self.delimiter) if self.is_multi_label() else [input_text]
-            y_pred.append([normalize_text(pred) for pred in predictions if pred])
-        labels: List[str] = list(set(y for ys in y_true for y in ys))
-        mlb = MultiLabelBinarizer().fit([labels])
+            y_pred.append([_normalize_label_text(pred) for pred in predictions if pred])
+        mlb = MultiLabelBinarizer().fit(
+            [[_normalize_label_text(label) for label in self.labels]] if self.labels else y_true
+        )
         y_true = mlb.transform(y_true)
         y_pred = mlb.transform(y_pred)
-        return [
-            Stat(MetricName("classification_macro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="macro")),
-            Stat(MetricName("classification_micro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="micro")),
-        ]
-
-
-class MultipleChoiceClassificationMetric(Metric):
+        stats: List[Stat] = []
+        for average in self.averages:
+            for score_name in self.scores:
+                if score_name == "f1":
+                    score_value = f1_score(y_pred=y_pred, y_true=y_true, average=average)
+                elif score_name == "precision":
+                    score_value = precision_score(y_pred=y_pred, y_true=y_true, average=average)
+                elif score_name == "recall":
+                    score_value = recall_score(y_pred=y_pred, y_true=y_true, average=average)
+                else:
+                    raise ValueError(
+                        f"Unknown score name: '{score_name}' - expected one of ['f1', 'precision', 'recall']"
+                    )
+                if average is None:
+                    for mlb_class, class_score_value in zip(mlb.classes_, score_value):
+                        stats.append(
+                            Stat(MetricName(f"classification_{mlb_class}_{score_name}")).add(class_score_value)
+                        )
+                else:
+                    stats.append(Stat(MetricName(f"classification_{average}_{score_name}")).add(score_value))
+        return stats
+
+
+class MultipleChoiceClassificationMetric(EvaluateInstancesMetric):
     """
     Calculate population micro/macro F1 score for multiple_choice_* adapters.
     For generation adapters, please use ClassificationMetric.
     """
 
-    def evaluate_instances(self, request_states: List[RequestState]) -> List[Stat]:
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
         y_pred: List[str] = []
         y_true: List[str] = []
         for request_state in request_states:  # one request state per instance
@@ -89,7 +154,9 @@ class MultipleChoiceClassificationMetric(Metric):
             ]
             assert len(golds) > 0, "MultipleChoiceClassificationMetric are designed for multiple_choice_* adapters"
             assert request_state.result is not None
-            sorted_completions: List[Sequence] = sorted(request_state.result.completions, key=lambda x: -x.logprob)
+            sorted_completions: List[GeneratedOutput] = sorted(
+                request_state.result.completions, key=lambda x: -x.logprob
+            )
             pred: str = sorted_completions[0].text.strip()  # Only utilize the first prediction
             if request_state.output_mapping is not None:
                 pred = request_state.output_mapping.get(pred, pred)
@@ -101,3 +168,21 @@ class MultipleChoiceClassificationMetric(Metric):
             Stat(MetricName("classification_macro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="macro")),
             Stat(MetricName("classification_micro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="micro")),
         ]
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="classification_macro_f1",
+                display_name="Macro F1",
+                description="Macro F1",
+                lower_is_better=False,
+                group="classification_metrics",
+            ),
+            MetricMetadata(
+                name="classification_micro_f1",
+                display_name="Micro F1",
+                description="Population-level micro-averaged F1 score.",
+                lower_is_better=False,
+                group="classification_metrics",
+            ),
+        ]

helm/benchmark/metrics/cleva_accuracy_metrics.py

@@ -3,12 +3,13 @@ from typing import List
 import numpy as np
 
 from helm.benchmark.adaptation.request_state import RequestState
-from helm.benchmark.metrics.metric import Metric, MetricName
+from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
+from helm.benchmark.metrics.metric import MetricName
 from helm.benchmark.metrics.statistic import Stat
-from helm.common.request import Sequence
+from helm.common.request import GeneratedOutput
 
 
-class CLEVATopKAccuracyMetric(Metric):
+class CLEVATopKAccuracyMetric(EvaluateInstancesMetric):
     """Defines metrics for CLEVA conceptual generalization task.
 
     This is not a conventional accuracy@k metric but rather a special one taken from
@@ -36,14 +37,16 @@ class CLEVATopKAccuracyMetric(Metric):
                             return True
         return False
 
-    def evaluate_instances(self, request_states: List[RequestState]) -> List[Stat]:
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
         per_instance_accuracy: List[bool] = []
         for request_state in request_states:  # one request state per instance
             assert request_state.result is not None
             references = request_state.instance.all_correct_references
             correct_ref_texts = [ref.output.text for ref in references if ref.output.text]
 
-            sorted_completions: List[Sequence] = sorted(request_state.result.completions, key=lambda x: -x.logprob)
+            sorted_completions: List[GeneratedOutput] = sorted(
+                request_state.result.completions, key=lambda x: -x.logprob
+            )
             sorted_completions_text: List[str] = [completion.text for completion in sorted_completions]
             correct = self.correct_or_not(sorted_completions_text, correct_ref_texts)
             per_instance_accuracy.append(correct)

helm/benchmark/metrics/cleva_harms_metrics.py

@@ -7,19 +7,20 @@ import numpy as np
 
 from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
 from helm.common.request import RequestResult
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hlog, hwarn
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.metrics.cleva_metrics_helper import ChineseTokenizer
-from helm.proxy.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
+from helm.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.optional_dependencies import handle_module_not_found_error
-from .bias_metrics import BiasMetric
-from .toxicity_metrics import ToxicityMetric
-from .copyright_metrics import BasicCopyrightMetric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.benchmark.metrics.bias_metrics import BiasMetric
+from helm.benchmark.metrics.toxicity_metrics import ToxicityMetric
+from helm.benchmark.metrics.copyright_metrics import BasicCopyrightMetric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.runner import get_benchmark_output_path
 
 try:
     import jieba
@@ -71,7 +72,7 @@ class CLEVABiasMetric(BiasMetric):
                   "demographic_category". One of "adjective" or "profession".
         """
         # Ensure all necessary data are downloaded
-        self.output_dir = os.path.join("benchmark_output", "metrics", self.name)
+        self.output_dir = os.path.join(get_benchmark_output_path(), "metrics", self.name)
         ensure_directory_exists(self.output_dir)
         for filename in self.FILE_NAMES:
             target_path = os.path.join(self.output_dir, filename)
@@ -136,7 +137,7 @@ class CLEVABiasMetric(BiasMetric):
         pair_to_count: Dict[Tuple[str, str], int] = defaultdict(int)
         for text in texts:
             tokens = jieba.lcut(text)  # Chinese word segmentation
-            for (target_word, group) in itertools.product(target_words, demographic_groups):
+            for target_word, group in itertools.product(target_words, demographic_groups):
                 group_words = self.demographic_group_to_words[group]
                 num_group_tokens = sum([tokens.count(w) for w in group_words])  # e.g. number of time asian names occur
                 num_target_tokens = tokens.count(target_word)  # e.g. number of tokens that are in the adj list
@@ -199,7 +200,7 @@ class CLEVAToxicityMetric(ToxicityMetric):
             )
         except PerspectiveAPIClientCredentialsError as e:
             self._perspective_api_unavailable = True
-            hlog(f"WARNING: Skipping ToxicityMetrics because Perspective API Client unavailable due to error: {e}")
+            hwarn(f"Skipping ToxicityMetrics because Perspective API Client unavailable due to error: {e}")
             hlog(
                 "To enable ToxicityMetrics, see: https://crfm-helm.readthedocs.io/en/latest/benchmark/#perspective-api"
             )

helm/benchmark/metrics/code_metrics.py

@@ -10,11 +10,11 @@ from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.scenarios.code_scenario import CodeReference
-from . import code_metrics_helper
-from .metric import Metric, MetricResult
-from .metric_service import MetricService
-from .metric_name import MetricName
-from .statistic import Stat
+from helm.benchmark.metrics import code_metrics_helper
+from helm.benchmark.metrics.metric import Metric, MetricResult
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
 
 MAXIMUM_MEMORY_BYTES = 8 * 1024 * 1024 * 1024  # 8GB.
 

helm/benchmark/metrics/code_metrics_helper.py

@@ -27,14 +27,24 @@ import signal
 import sys
 import tempfile
 from typing import List, Union, Dict, Optional
+from types import ModuleType
 from unittest.mock import patch, mock_open
 
 import numpy as np
-from pyext import RuntimeModule
 
 from helm.common.hierarchical_logger import hlog
 
 
+class RuntimeModule(ModuleType):
+    """crfm-helm's replacement for pyext.RuntimeModule, since pyext is not supported by Python >=3.11"""
+
+    @staticmethod
+    def from_string(module_name: str, module_doc: str, module_contents: str) -> "RuntimeModule":
+        module = RuntimeModule(module_name, module_doc)
+        exec(module_contents, module.__dict__)
+        return module
+
+
 # === APPS evaluation below ===
 class CodeType(Enum):
     call_based = 0
@@ -448,9 +458,7 @@ def check_correctness(problem: Dict, completion: str, timeout: float, completion
     """
 
     def unsafe_execute():
-
         with create_tempdir():
-
             # These system calls are needed when cleaning up tempdir.
             import os
             import shutil