crfm-helm 0.5.3__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/metrics/bias_metrics.py

@@ -8,9 +8,18 @@ from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMe
 
 from helm.common.request import RequestResult, GeneratedOutput
 from helm.benchmark.adaptation.request_state import RequestState
-from .statistic import Stat
-from .metric_name import MetricName
-from .bias_word_lists import GENDER_TO_WORD_LISTS, RACE_TO_NAME_LISTS, ADJECTIVE_LIST, PROFESSION_LIST
+from helm.benchmark.metrics.nltk_helper import install_nltk_resources
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.bias_word_lists import (
+    GENDER_TO_WORD_LISTS,
+    RACE_TO_NAME_LISTS,
+    ADJECTIVE_LIST,
+    PROFESSION_LIST,
+)
+
+
+install_nltk_resources()
 
 
 class BiasMetric(EvaluateInstancesMetric):

helm/benchmark/metrics/bigcodebench_metrics.py

@@ -0,0 +1,25 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class BigCodeBenchMetric(Metric):
+    """Score metrics for BigCodeBench."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        score = request_state.annotations["bigcodebench"]["pass_at_one"]
+        return [
+            Stat(MetricName("bigcodebench_p@1")).add(score),
+        ]

helm/benchmark/metrics/bird_sql_metrics.py

@@ -0,0 +1,28 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class BirdSQLMetric(Metric):
+    """Score metrics for Bird-SQL."""
+
+    ANNOTATOR_NAME = "bird_sql"
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        if not request_state.annotations:
+            raise Exception("Request state did not have annotations.")
+        predicted_result = request_state.annotations[self.ANNOTATOR_NAME]["predicted_result"]
+        ground_truth_result = request_state.annotations[self.ANNOTATOR_NAME]["ground_truth_result"]
+        execution_accuracy = int(set(predicted_result) == set(ground_truth_result))
+        return [Stat(MetricName("execution_accuracy")).add(execution_accuracy)]

helm/benchmark/metrics/chw_care_plan_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.chw_care_plan_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class CHWCarePlanMetric(Metric):
+    """Score metrics for CHWCarePla."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["chw_care_plan"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("chw_care_plan_accuracy")).add(score),
+        ]

helm/benchmark/metrics/classification_metrics.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from sklearn.metrics import f1_score
+from sklearn.metrics import f1_score, precision_score, recall_score
 from sklearn.preprocessing import MultiLabelBinarizer
 
 from helm.benchmark.adaptation.request_state import RequestState
@@ -9,9 +9,14 @@ from helm.benchmark.metrics.evaluate_reference_metrics import normalize_text
 from helm.benchmark.metrics.metric import MetricName
 from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.scenarios.scenario import Reference
+from helm.common.hierarchical_logger import hlog
 from helm.common.request import GeneratedOutput
 
 
+def _normalize_label_text(text: str) -> str:
+    return normalize_text(text, should_remove_articles=False)
+
+
 class ClassificationMetric(EvaluateInstancesMetric):
     """Defines metrics for multi-class classification using the generation adapter.
 
@@ -23,8 +28,12 @@ class ClassificationMetric(EvaluateInstancesMetric):
     reference. The predicted class for each instance is the normalized text of the generation.
 
     Note:
-    - The set of classes is derived from the correct references from all the instances.
-      This means that classes may be omitted if they are never used as a correct reference.
+    - It is highly recommended to specify the set of classes should be specified using the
+      `labels` parameter. Otherwise, the set of classes is derived from the correct references
+      from all the instances. This means that classes may be incorrectly omitted if they are never
+      used as a correct reference.
+    - The `averages` parameter is a list of averaging methods to be used.
+      It has the same meaning `average` as in scikit-learn.
     - Generations that are not in any of the known classes are counted as a
       negative prediction for every class.
     - Perturbed classes are considered different classes from unperturbed
@@ -32,8 +41,45 @@ class ClassificationMetric(EvaluateInstancesMetric):
     - Currently, multi-label classification is not supported.
     """
 
-    def __init__(self, delimiter: Optional[str] = None):
+    AVERAGE_OPTIONS = ["micro", "macro", "weighted", None]
+    SCORE_OPTIONS = ["f1", "precision", "recall"]
+
+    def __init__(
+        self,
+        averages: Optional[List[Optional[str]]] = None,
+        labels: Optional[List[str]] = None,
+        scores: Optional[List[str]] = None,
+        delimiter: Optional[str] = None,
+    ) -> None:
+        """Creates metrics for multi-class classification.
+
+        :param delimiter: For multi-label classification, the string delimiter between classes in the model's output.
+        :param average: The list of scores to compute (e.g. "f1", "precision", "recall").
+          Defaults to ["f1"].
+        :param average: The averaging methods (e.g. "micro", "macro", "weighted") to be used.
+          It has the same meaning `average` as in scikit-learn.
+          Defaults to ["macro", "micro"].
+        :param labels: The set of labels.
+        :return: A list of `Stat` objects.
+        """
+        self.averages = averages or ["macro", "micro"]
+        for average in self.averages:
+            if average not in ClassificationMetric.AVERAGE_OPTIONS:
+                raise ValueError(
+                    f"Each value in `averages` must be set to one of {ClassificationMetric.AVERAGE_OPTIONS}."
+                )
+        self.scores = scores or ["f1"]
+        for score_name in self.scores:
+            if score_name not in ClassificationMetric.SCORE_OPTIONS:
+                raise ValueError(f"Each value in `scores` must be set to one of {ClassificationMetric.SCORE_OPTIONS}.")
         self.delimiter = delimiter
+        self.labels = labels
+        if not self.labels:
+            hlog(
+                "WARNING: `labels` were not set on `ClassificationMetric`, "
+                "so they will be inferred from target references. "
+                "It is recommend to explicitly set `labels` on `ClassificationMetric`."
+            )
 
     def is_multi_label(self) -> bool:
         return bool(self.delimiter)
@@ -57,20 +103,38 @@ class ClassificationMetric(EvaluateInstancesMetric):
             references = request_state.instance.all_correct_references
             if not self.is_multi_label():
                 assert len(references) == 1
-            correct_ref_texts = [normalize_text(ref.output.text) for ref in references if ref.output.text]
+            correct_ref_texts = [_normalize_label_text(ref.output.text) for ref in references if ref.output.text]
             y_true.append(correct_ref_texts)
 
             input_text = request_state.result.completions[0].text
             predictions = input_text.split(self.delimiter) if self.is_multi_label() else [input_text]
-            y_pred.append([normalize_text(pred) for pred in predictions if pred])
-        labels: List[str] = list(set(y for ys in y_true for y in ys))
-        mlb = MultiLabelBinarizer().fit([labels])
+            y_pred.append([_normalize_label_text(pred) for pred in predictions if pred])
+        mlb = MultiLabelBinarizer().fit(
+            [[_normalize_label_text(label) for label in self.labels]] if self.labels else y_true
+        )
         y_true = mlb.transform(y_true)
         y_pred = mlb.transform(y_pred)
-        return [
-            Stat(MetricName("classification_macro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="macro")),
-            Stat(MetricName("classification_micro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="micro")),
-        ]
+        stats: List[Stat] = []
+        for average in self.averages:
+            for score_name in self.scores:
+                if score_name == "f1":
+                    score_value = f1_score(y_pred=y_pred, y_true=y_true, average=average)
+                elif score_name == "precision":
+                    score_value = precision_score(y_pred=y_pred, y_true=y_true, average=average)
+                elif score_name == "recall":
+                    score_value = recall_score(y_pred=y_pred, y_true=y_true, average=average)
+                else:
+                    raise ValueError(
+                        f"Unknown score name: '{score_name}' - expected one of ['f1', 'precision', 'recall']"
+                    )
+                if average is None:
+                    for mlb_class, class_score_value in zip(mlb.classes_, score_value):
+                        stats.append(
+                            Stat(MetricName(f"classification_{mlb_class}_{score_name}")).add(class_score_value)
+                        )
+                else:
+                    stats.append(Stat(MetricName(f"classification_{average}_{score_name}")).add(score_value))
+        return stats
 
 
 class MultipleChoiceClassificationMetric(EvaluateInstancesMetric):

helm/benchmark/metrics/cleva_harms_metrics.py

@@ -14,12 +14,13 @@ from helm.benchmark.metrics.cleva_metrics_helper import ChineseTokenizer
 from helm.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists
 from helm.common.optional_dependencies import handle_module_not_found_error
-from .bias_metrics import BiasMetric
-from .toxicity_metrics import ToxicityMetric
-from .copyright_metrics import BasicCopyrightMetric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.benchmark.metrics.bias_metrics import BiasMetric
+from helm.benchmark.metrics.toxicity_metrics import ToxicityMetric
+from helm.benchmark.metrics.copyright_metrics import BasicCopyrightMetric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.runner import get_benchmark_output_path
 
 try:
     import jieba
@@ -71,7 +72,7 @@ class CLEVABiasMetric(BiasMetric):
                   "demographic_category". One of "adjective" or "profession".
         """
         # Ensure all necessary data are downloaded
-        self.output_dir = os.path.join("benchmark_output", "metrics", self.name)
+        self.output_dir = os.path.join(get_benchmark_output_path(), "metrics", self.name)
         ensure_directory_exists(self.output_dir)
         for filename in self.FILE_NAMES:
             target_path = os.path.join(self.output_dir, filename)

helm/benchmark/metrics/code_metrics.py

@@ -10,11 +10,11 @@ from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.scenarios.code_scenario import CodeReference
-from . import code_metrics_helper
-from .metric import Metric, MetricResult
-from .metric_service import MetricService
-from .metric_name import MetricName
-from .statistic import Stat
+from helm.benchmark.metrics import code_metrics_helper
+from helm.benchmark.metrics.metric import Metric, MetricResult
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
 
 MAXIMUM_MEMORY_BYTES = 8 * 1024 * 1024 * 1024  # 8GB.
 

helm/benchmark/metrics/comet_metric.py

@@ -0,0 +1,125 @@
+import logging
+from typing import List
+
+import comet
+from torch import nn
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.metrics.metric import Metric, MetricResult
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hlog
+from helm.common.request import RequestResult
+
+
+class CometMetric(Metric):
+    """COMET machine translation metric using a regression model.
+    The model takes a triplet of source sentence, translation, and reference
+    and computes a score in the range [0, 1] reflecting the quality of the predicted
+    translation.
+
+    Paper:
+    @inproceedings{rei-etal-2022-comet,
+        title = "{COMET}-22: Unbabel-{IST} 2022 Submission for the Metrics Shared Task",
+        author = "Rei, Ricardo  and
+            C. de Souza, Jos{\'e} G.  and
+            Alves, Duarte  and
+            Zerva, Chrysoula  and
+            Farinha, Ana C  and
+            Glushkova, Taisiya  and
+            Lavie, Alon  and
+            Coheur, Luisa  and
+            Martins, Andr{\'e} F. T.",
+            editor = {Koehn, Philipp  and
+            Barrault, Lo{\"\i}c  and
+            Bojar, Ond{\v{r}}ej  and
+            Bougares, Fethi  and
+            Chatterjee, Rajen  and
+            Costa-juss{\`a}, Marta R.  and
+            Federmann, Christian  and
+            Fishel, Mark  and
+            Fraser, Alexander  and
+            Freitag, Markus  and
+            Graham, Yvette  and
+            Grundkiewicz, Roman  and
+            Guzman, Paco  and
+            Haddow, Barry  and
+            Huck, Matthias  and
+            Jimeno Yepes, Antonio  and
+            Kocmi, Tom  and
+            Martins, Andr{\'e}  and
+            Morishita, Makoto  and
+            Monz, Christof  and
+            Nagata, Masaaki  and
+            Nakazawa, Toshiaki  and
+            Negri, Matteo  and
+            N{\'e}v{\'e}ol, Aur{\'e}lie  and
+            Neves, Mariana  and
+            Popel, Martin  and
+            Turchi, Marco  and
+            Zampieri, Marcos},
+        booktitle = "Proceedings of the Seventh Conference on Machine Translation (WMT)",
+        month = dec,
+        year = "2022",
+        publisher = "Association for Computational Linguistics",
+        url = "https://aclanthology.org/2022.wmt-1.52",
+    }
+    """
+
+    METRIC_NAME = "comet"
+
+    def __init__(self, task: str, model_name: str = "Unbabel/wmt22-comet-da", device: str = "cpu"):
+        self.model_name = model_name
+        self.comet_scorer: nn.Module = self._load_model(model_name)
+        self.num_gpus = 0 if device == "cpu" else 1
+
+        # suppress warnings from PyTorch Lightning which spams terminal
+        logging.getLogger("lightning.pytorch.utilities.rank_zero").setLevel(logging.WARNING)
+        logging.getLogger("lightning.pytorch.accelerators.cuda").setLevel(logging.WARNING)
+
+    @staticmethod
+    def _load_model(model_name: str) -> nn.Module:
+        """Load Comet model from the checkpoint.
+
+        Returns:
+            The loaded model.
+        """
+        return comet.load_from_checkpoint(comet.download_model(model_name))
+
+    def evaluate(
+        self, scenario_state: ScenarioState, metric_service: MetricService, eval_cache_path: str, parallelism: int
+    ) -> MetricResult:
+        hlog(
+            f"Setting parallelism from {parallelism} to 1, since "
+            f"evaluating {self.__class__.__name__} with parallelism > 1 seg faults."
+        )
+        return super().evaluate(scenario_state, metric_service, eval_cache_path, parallelism=1)
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """Compute the COMET score for this instance"""
+        assert len(request_state.instance.references) == 1
+        ref = request_state.instance.references[0].output.text
+        src = request_state.instance.input.text
+
+        result = request_state.result
+        if not isinstance(result, RequestResult):
+            raise TypeError(f"Expected a valid result, but got {result}!")
+        mt = result.completions[0].text.strip()
+
+        # comet requires this exac5 format
+        data = [dict(ref=ref, src=src, mt=mt)]
+        output = self.comet_scorer.predict(data, gpus=self.num_gpus, progress_bar=False)  # type: ignore
+        comet_score = output[0][0]  # extract the actual score
+
+        metric_result = [Stat(MetricName(self.METRIC_NAME)).add(comet_score)]
+
+        return metric_result

helm/benchmark/metrics/common_metric_specs.py

@@ -44,11 +44,18 @@ def get_language_modeling_metric_specs(names: List[str]) -> List[MetricSpec]:
     ]
 
 
-def get_classification_metric_specs(delimiter: Optional[str] = None) -> List[MetricSpec]:
+def get_classification_metric_specs(
+    labels: Optional[List[str]] = None, delimiter: Optional[str] = None
+) -> List[MetricSpec]:
+    extra_args: Dict[str, Any] = {}
+    if labels:
+        extra_args["labels"] = labels
+    if delimiter:
+        extra_args["delimiter"] = delimiter
     return [
         MetricSpec(
             class_name="helm.benchmark.metrics.classification_metrics.ClassificationMetric",
-            args={"delimiter": delimiter},
+            args=extra_args,
         )
     ]
 

helm/benchmark/metrics/conv_fin_qa_calc_metrics.py

@@ -0,0 +1,72 @@
+import re
+from typing import Any, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+from helm.common.hierarchical_logger import hlog
+
+
+def _strip_string(str: str) -> Any:
+    # from https://stackoverflow.com/a/4703508
+    numeric_const_pattern = r"[-+]?(?:(?:\d*\.\d+)|(?:\d+\.?))(?:[Ee][+-]?\d+)?"
+    match = re.search(numeric_const_pattern, str)
+    if match:
+        try:
+            return float(str[match.start() : match.end()])
+        except Exception:
+            return None
+    return None
+
+
+def float_equiv(str1: str, str2: str, eps: float = 1e-6) -> float:
+    """Check if two values have the same float value, up to a small tolerance.
+
+    This is the implementation used in the IBM Enterprise Benchmark paper.
+
+    Note: This is a "mostly-correct" equality function and does not handle some cases correctly:
+
+    - If both values are non-floats, then it will always return 1.0,
+      regardless of whether strings match.
+    - If either of both values have different units (e.g. currency symbols,
+      trailing "M" or "B", trailing %), the values will not be converted to the same
+      units before comparison.
+    """
+    try:
+        ss1 = _strip_string(str1)
+        ss2 = _strip_string(str2)
+
+        if ss1 is None or ss2 is None:
+            hlog("WARNING: float_equiv returning 1.0 because both values are non-floats")
+            return 0.0
+        return float(abs(ss1 - ss2) < eps)
+    except Exception:
+        return float(str1 == str2)
+
+
+class ConvFinQACalcMetric(Metric):
+    """Score metrics for AIRBench 2024."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        model_answer = request_state.result.completions[0].text
+
+        assert len(request_state.instance.references) == 1
+        assert len(request_state.instance.references[0].tags) == 1
+        assert request_state.instance.references[0].tags[0] == CORRECT_TAG
+        gold_answer = request_state.instance.references[0].output.text
+
+        return [
+            Stat(MetricName("float_equiv")).add(float_equiv(model_answer, gold_answer)),
+        ]

helm/benchmark/metrics/copyright_metrics.py

@@ -9,10 +9,10 @@ from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.benchmark.scenarios.scenario import Reference
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import RequestResult
-from .metric import Metric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
 
 try:
     import numba

helm/benchmark/metrics/czech_bank_qa_metrics.py

@@ -0,0 +1,29 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class CzechBankQAMetrics(Metric):
+    """Score metrics for AIRBench 2024."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        # assert len(request_state.instance.references) > 1
+        # category_text = request_state.instance.references[0].output.text
+        # category_parts = category_text.split(".")
+        # assert len(category_parts) == 3
+        assert request_state.annotations
+        error_rate = 0.0 if request_state.annotations["czech_bank_qa"]["error"] is None else 1.0
+        return [
+            Stat(MetricName("error_rate")).add(error_rate),
+        ]

helm/benchmark/metrics/decodingtrust_fairness_metrics.py

@@ -1,8 +1,8 @@
 from typing import List
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
-from .metric_name import MetricName
-from .statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.metrics.evaluate_reference_metrics import normalize_text
 import numpy as np
 from fairlearn.metrics import equalized_odds_difference

helm/benchmark/metrics/decodingtrust_privacy_metrics.py

@@ -1,7 +1,7 @@
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
-from .metric_name import MetricName
-from .statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
 from typing import List
 import re
 

helm/benchmark/metrics/decodingtrust_stereotype_bias_metrics.py

@@ -3,8 +3,8 @@ from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMe
 from helm.common.request import GeneratedOutput
 from helm.common.request import RequestResult
 from helm.benchmark.adaptation.request_state import RequestState
-from .metric_name import MetricName
-from .statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
 
 import numpy as np
 

helm/benchmark/metrics/dischargeme_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.dischargeme_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class DischargeMeMetric(Metric):
+    """Score metrics for DischargeMe."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["dischargeme"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("dischargeme_accuracy")).add(score),
+        ]

helm/benchmark/metrics/disinformation_metrics.py

@@ -11,10 +11,10 @@ from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.common.request import RequestResult, GeneratedOutput
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from .metric import Metric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
 
 try:
     from sacrebleu.metrics import BLEU

helm/benchmark/metrics/dry_run_metrics.py

@@ -8,11 +8,11 @@ from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.statistic import Stat, merge_stat
 from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
-from .metric import MetricInterface, MetricResult, PerInstanceStats
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .tokens.auto_token_cost_estimator import AutoTokenCostEstimator
-from .tokens.token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.tokens.auto_token_cost_estimator import AutoTokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 @dataclass