crfm-helm 0.2.0__py3-none-any.whl → 0.2.2__py3-none-any.whl

helm/benchmark/metrics/classification_metrics.py

@@ -0,0 +1,70 @@
+from typing import List, Optional
+
+from sklearn.metrics import f1_score
+from sklearn.preprocessing import MultiLabelBinarizer
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.basic_metrics import normalize_text
+from helm.benchmark.metrics.metric import Metric, MetricName
+from helm.benchmark.metrics.statistic import Stat
+
+
+class ClassificationMetric(Metric):
+    """Defines metrics for multi-class classification using the generation adapter.
+
+    Currently provides `classification_macro_f1` and `classification_micro_f1`.
+    These are population-level F1 measures to measure classification performance where each
+    generation is a predicted class, and are different from the instance-level F1 measures
+    in `BasicMetrics` that are intended to measure word overlap between the correct references
+    and generations. The correct class should be provided by the normalized text of a correct
+    reference. The predicted class for each instance is the normalized text of the generation.
+
+    Note:
+    - The set of classes is derived from the correct references from all the instances.
+      This means that classes may be omitted if they are never used as a correct reference.
+    - Generations that are not in any of the known classes are counted as a
+      negative prediction for every class.
+    - Perturbed classes are considered different classes from unperturbed
+      classes.
+    - Currently, multi-label classification is not supported.
+    """
+
+    def __init__(self, delimiter: Optional[str] = None):
+        self.delimiter = delimiter
+
+    def is_multi_label(self) -> bool:
+        return bool(self.delimiter)
+
+    def evaluate_instances(self, request_states: List[RequestState]) -> List[Stat]:
+        y_pred: List[List[str]] = []
+        y_true: List[List[str]] = []
+        for request_state in request_states:  # one request state per instance
+            # Only the generation adapter is supported.
+            # TODO: Support multiple_choice_* adapters.
+            if request_state.reference_index is not None:
+                raise ValueError("ClassificationMetric does not support multiple choice separate adapters")
+            if request_state.request_mode == "calibration":
+                raise ValueError("ClassificationMetric does not support calibration requests")
+            assert request_state.result is not None
+            if len(request_state.result.completions) != 1:
+                raise ValueError("Result must contain exactly one completion")
+            if request_state.output_mapping:
+                raise ValueError("ClassificationMetric does not support multiple choice adapters")
+
+            references = request_state.instance.all_correct_references
+            if not self.is_multi_label():
+                assert len(references) == 1
+            correct_ref_texts = [normalize_text(ref.output.text) for ref in references if ref.output.text]
+            y_true.append(correct_ref_texts)
+
+            input_text = request_state.result.completions[0].text
+            predictions = input_text.split(self.delimiter) if self.is_multi_label() else [input_text]
+            y_pred.append([normalize_text(pred) for pred in predictions if pred])
+        labels: List[str] = list(set(y for ys in y_true for y in ys))
+        mlb = MultiLabelBinarizer().fit([labels])
+        y_true = mlb.transform(y_true)
+        y_pred = mlb.transform(y_pred)
+        return [
+            Stat(MetricName("classification_macro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="macro")),
+            Stat(MetricName("classification_micro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="micro")),
+        ]

helm/benchmark/metrics/machine_translation_metrics.py

@@ -0,0 +1,36 @@
+from typing import List
+from sacrebleu import BLEU
+
+from helm.benchmark.adaptation.request_state import RequestState
+from .metric import Metric
+from .metric_name import MetricName
+from .statistic import Stat
+
+
+class MachineTranslationMetric(Metric):
+    """
+    Compute the BLEU score for Machine Translation scenarios. The implementation is based on sacrebleu.
+    """
+
+    def evaluate_instances(self, request_states: List[RequestState]) -> List[Stat]:
+        """
+        Compute the corpus-level metric based on all reqeust_states.
+        """
+
+        bleu = BLEU()
+
+        refs: List[List[str]] = [[]]
+        sys: List = []
+        for request_state in request_states:
+            # Assume there is one referece per instance. TODO: Support multiple references after adding more scenarios.
+            num_references: int = len(request_state.instance.references)
+            if num_references != 1:
+                raise ValueError(f"This instance has {num_references} references, but we currently only support one.")
+            # Usually there is only one completion for each instance.
+            assert request_state.result is not None
+            if len(request_state.result.completions) != 1:
+                raise ValueError("Each request result should have only exactly one completion.")
+            sys.append(request_state.result.completions[0].text)
+            refs[0].append(request_state.instance.references[0].output.text)
+        bleu_score = bleu.corpus_score(sys, refs).score
+        return [Stat(MetricName("bleu")).add(bleu_score)]

helm/benchmark/metrics/summarization_metrics.py

@@ -5,17 +5,9 @@ import os
 import pickle
 
 import spacy
-import subprocess
-import sys
 from typing import List, Dict, Optional
 from collections import defaultdict
 
-# Need to check spacy module is downloaded before importing DataStatsMetric
-if not spacy.util.is_package("en_core_web_sm"):
-    subprocess.check_call([sys.executable, "-m", "spacy", "download", "en_core_web_sm"])
-
-from summ_eval.data_stats_metric import DataStatsMetric
-
 from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
@@ -56,6 +48,13 @@ class SummarizationMetric(Metric):
             "rouge_2": get_rouge_function("rouge2"),
             "rouge_l": get_rouge_function("rougeL"),
         }
+        # Download en_core_web_sm before importing DataStatsMetric to
+        # avoid triggering a bug in DataStatsMetric that raises
+        # `NameError: name 'stderr' is not defined`
+        if not spacy.util.is_package("en_core_web_sm"):
+            spacy.cli.download("en_core_web_sm")  # type: ignore
+        from summ_eval.data_stats_metric import DataStatsMetric
+
         self.data_stats_metric = DataStatsMetric()
         self.task: str = task
         self.qa_fact_eval: Optional[Dict] = None

helm/benchmark/metrics/test_classification_metrics.py

@@ -0,0 +1,150 @@
+from collections import defaultdict
+from typing import Dict, List, NamedTuple
+
+from pytest import approx
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.classification_metrics import ClassificationMetric
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.scenarios.scenario import Input, Instance, Output, Reference, CORRECT_TAG
+from helm.common.request import Request, RequestResult, Sequence
+
+
+class _Option(NamedTuple):
+    text: str
+    is_correct: bool
+
+
+def _request_state(prediction: str, options: List[_Option]):
+    references = [
+        Reference(output=Output(text=option.text), tags=[CORRECT_TAG] if option.is_correct else [])
+        for option in options
+    ]
+    return RequestState(
+        instance=Instance(input=Input(text=""), references=references),
+        reference_index=None,
+        request_mode=None,
+        train_trial_index=0,
+        output_mapping=None,
+        request=Request(),
+        result=RequestResult(
+            success=True, embedding=[], completions=[Sequence(text=prediction, logprob=0.0, tokens=[])], cached=False
+        ),
+        num_train_instances=0,
+        prompt_truncated=False,
+    )
+
+
+def assert_stats_equal(actual_stats: List[Stat], expected_values: Dict[str, float]):
+    actual_values = {stat.name.name: stat.mean for stat in actual_stats}
+    assert actual_values == approx(expected_values)
+
+
+def _expected_stats(all_classes_counts: Dict[str, Dict[str, int]]):
+    micro_counts: Dict[str, int] = defaultdict(int)
+    for class_counts in all_classes_counts.values():
+        for key, class_count in class_counts.items():
+            micro_counts[key] += class_count
+    micro_precision = micro_counts["tp"] / (micro_counts["tp"] + micro_counts["fp"])
+    micro_recall = micro_counts["tp"] / (micro_counts["tp"] + micro_counts["fn"])
+    micro_f1 = 2 * (micro_precision * micro_recall) / (micro_precision + micro_recall)
+
+    class_f1: List[float] = []
+    for class_counts in all_classes_counts.values():
+        class_precision = class_counts["tp"] / (class_counts["tp"] + class_counts["fp"])
+        class_recall = class_counts["tp"] / (class_counts["tp"] + class_counts["fn"])
+        class_f1.append(2 * (class_precision * class_recall) / (class_precision + class_recall))
+    macro_f1 = sum(class_f1) / len(class_f1)
+
+    return {
+        "classification_micro_f1": micro_f1,
+        "classification_macro_f1": macro_f1,
+    }
+
+
+def test_evaluate_instances_binary_generation():
+    metric = ClassificationMetric(delimiter=None)
+
+    request_states = [
+        _request_state("yes", [_Option("yes", True)]),
+        _request_state("yes", [_Option("yes", True)]),
+        _request_state("yes", [_Option("yes", True)]),
+        _request_state("yes", [_Option("no", True)]),
+        _request_state("no", [_Option("yes", True)]),
+        _request_state("no", [_Option("no", True)]),
+        _request_state("invalid", [_Option("no", True)]),
+    ]
+
+    assert_stats_equal(
+        metric.evaluate_instances(request_states),
+        _expected_stats(
+            {
+                "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
+                "no": {"tp": 1, "fp": 1, "tn": 3, "fn": 2},
+            }
+        ),
+    )
+
+
+def test_evaluate_instances_multi_class():
+    # Note: no "a" because it would get filtered out by normalize_text()
+    metric = ClassificationMetric(delimiter=None)
+
+    def _options(correct: str):
+        return [_Option(text, text == correct) for text in ["d", "b", "c"]]
+
+    request_states = [
+        _request_state("d", _options("d")),
+        _request_state("d", _options("d")),
+        _request_state("d", _options("d")),
+        _request_state("d", _options("b")),
+        _request_state("b", _options("b")),
+        _request_state("b", _options("b")),
+        _request_state("b", _options("c")),
+        _request_state("c", _options("d")),
+        _request_state("c", _options("c")),
+        _request_state("invalid", _options("c")),
+    ]
+    assert_stats_equal(
+        metric.evaluate_instances(request_states),
+        _expected_stats(
+            {
+                "d": {"tp": 3, "fp": 1, "tn": 5, "fn": 1},
+                "b": {"tp": 2, "fp": 1, "tn": 6, "fn": 1},
+                "c": {"tp": 1, "fp": 1, "tn": 6, "fn": 2},
+            }
+        ),
+    )
+
+
+def test_evaluate_instances_multilabel():
+    # Note: no "a" because it would get filtered out by normalize_text()
+    metric = ClassificationMetric(delimiter=",")
+
+    def _options(correct: List[str]):
+        return [_Option(text, text in correct) for text in ["d", "b", "c"]]
+
+    request_states = [
+        _request_state("d,b", _options(["d", "b"])),
+        _request_state("d,b", _options(["d", "c"])),
+        _request_state("d", _options(["d"])),
+        _request_state("c", _options(["b"])),
+        _request_state("b", _options(["b", "c"])),
+        _request_state("d,b", _options(["c"])),
+        _request_state("d,c", _options(["d"])),
+        _request_state("d,b,c", _options(["d", "b", "c"])),
+        _request_state("", []),
+        _request_state("n/a", []),
+        _request_state("invalid", _options(["c"])),
+    ]
+
+    assert_stats_equal(
+        metric.evaluate_instances(request_states),
+        _expected_stats(
+            {
+                "d": {"tp": 5, "fp": 1, "tn": 5, "fn": 0},
+                "b": {"tp": 3, "fp": 2, "tn": 5, "fn": 1},
+                "c": {"tp": 1, "fp": 2, "tn": 4, "fn": 4},
+            }
+        ),
+    )