crfm-helm 0.5.4__py3-none-any.whl → 0.5.5__py3-none-any.whl

helm/benchmark/augmentations/contrast_sets_perturbation.py

@@ -4,8 +4,8 @@ from random import Random
 from typing import List, Optional
 
 from helm.benchmark.scenarios.scenario import Instance, Reference, Input
-from .perturbation_description import PerturbationDescription
-from .perturbation import Perturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import Perturbation
 
 
 class ContrastSetsPerturbation(Perturbation):

helm/benchmark/augmentations/dialect_perturbation.py

@@ -7,8 +7,9 @@ from pathlib import Path
 from typing import Dict, Optional, List
 
 from helm.common.general import match_case, ensure_file_downloaded
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.runner import get_benchmark_output_path
 
 
 class DialectPerturbation(TextPerturbation):
@@ -20,7 +21,7 @@ class DialectPerturbation(TextPerturbation):
     should_perturb_references: bool = True
 
     """ Output path to store external files and folders """
-    OUTPUT_PATH = os.path.join("benchmark_output", "perturbations", name)
+    OUTPUT_PATH = os.path.join(get_benchmark_output_path(), "perturbations", name)
 
     """ Dictionary mapping dialects to one another """
     SAE = "SAE"
@@ -74,8 +75,6 @@ class DialectPerturbation(TextPerturbation):
                 self.MAPPING_DICTS for the provided source and target classes
                 will be used, if available.
         """
-        # TODO: Update path so it is not hard-coded to benchmark_output
-        # https://github.com/stanford-crfm/benchmarking/issues/493
         self.output_path: str = self.OUTPUT_PATH
         Path(self.output_path).mkdir(parents=True, exist_ok=True)
 

helm/benchmark/augmentations/extra_space_perturbation.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from random import Random
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class ExtraSpacePerturbation(TextPerturbation):

helm/benchmark/augmentations/filler_words_perturbation.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 from random import Random
 

helm/benchmark/augmentations/gender_perturbation.py

@@ -5,8 +5,8 @@ import re
 from typing import Dict, List, Optional, Tuple
 
 from helm.common.general import match_case
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
 
 
 """ Gender term mappings """

helm/benchmark/augmentations/lowercase_perturbation.py

@@ -1,7 +1,7 @@
 from random import Random
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class LowerCasePerturbation(TextPerturbation):

helm/benchmark/augmentations/mild_mix_perturbation.py

@@ -1,11 +1,11 @@
 from random import Random
 
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
-from .lowercase_perturbation import LowerCasePerturbation
-from .contraction_expansion_perturbation import ContractionPerturbation
-from .space_perturbation import SpacePerturbation
-from .misspelling_perturbation import MisspellingPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.lowercase_perturbation import LowerCasePerturbation
+from helm.benchmark.augmentations.contraction_expansion_perturbation import ContractionPerturbation
+from helm.benchmark.augmentations.space_perturbation import SpacePerturbation
+from helm.benchmark.augmentations.misspelling_perturbation import MisspellingPerturbation
 
 
 class MildMixPerturbation(TextPerturbation):

helm/benchmark/augmentations/misspelling_perturbation.py

@@ -6,8 +6,8 @@ from random import Random
 from typing import Dict, List
 
 from helm.common.general import match_case
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 # The implementation below is based on the following list of common misspellings:

helm/benchmark/augmentations/person_name_perturbation.py

@@ -9,8 +9,9 @@ from typing import Dict, List, Optional, Set
 
 from helm.benchmark.scenarios.scenario import Input, Instance, Reference, Output
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists, match_case
-from .perturbation_description import PerturbationDescription
-from .perturbation import Perturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import Perturbation
+from helm.benchmark.runner import get_benchmark_output_path
 
 
 # Pull this out so serialization works for multiprocessing
@@ -35,7 +36,7 @@ class PersonNamePerturbation(Perturbation):
         "https://storage.googleapis.com/crfm-helm-public/source_datasets/"
         "augmentations/person_name_perturbation/person_names.txt"
     )
-    OUTPUT_PATH = os.path.join("benchmark_output", "perturbations", name)
+    OUTPUT_PATH = os.path.join(get_benchmark_output_path(), "perturbations", name)
 
     """ Name types """
     FIRST_NAME = "first_name"
@@ -153,8 +154,6 @@ class PersonNamePerturbation(Perturbation):
                 find the gender association for a source_word, we randomly
                 pick from one of the target names.
         """
-        # TODO: Update path so it is not hard-coded to benchmark_output
-        # https://github.com/stanford-crfm/benchmarking/issues/493
         self.output_path: str = self.OUTPUT_PATH
         Path(self.output_path).mkdir(parents=True, exist_ok=True)
 

helm/benchmark/augmentations/perturbation.py

@@ -4,7 +4,7 @@ from random import Random
 from typing import List, Optional
 
 
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 from helm.benchmark.scenarios.scenario import Input, Instance, Reference, Output
 from helm.common.object_spec import ObjectSpec, create_object
 

helm/benchmark/augmentations/space_perturbation.py

@@ -2,8 +2,8 @@ from dataclasses import dataclass
 from random import Random
 import re
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class SpacePerturbation(TextPerturbation):

helm/benchmark/augmentations/suffix_perturbation.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from random import Random
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class SuffixPerturbation(TextPerturbation):

helm/benchmark/augmentations/synonym_perturbation.py

@@ -10,8 +10,9 @@ from nltk.corpus import wordnet
 import spacy
 
 from helm.common.general import match_case, ensure_file_downloaded
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.runner import get_benchmark_output_path
 
 
 class SynonymPerturbation(TextPerturbation):
@@ -57,7 +58,7 @@ class SynonymPerturbation(TextPerturbation):
             spacy.cli.download("en_core_web_sm")  # type: ignore
             self.spacy_model = spacy.load("en_core_web_sm")
 
-        output_dir = os.path.join("benchmark_output", "perturbations", self.name)
+        output_dir = os.path.join(get_benchmark_output_path(), "perturbations", self.name)
         Path(output_dir).mkdir(parents=True, exist_ok=True)
         nltk.data.path.append(output_dir)
         try:

helm/benchmark/augmentations/test_perturbation.py

@@ -4,19 +4,22 @@ import unittest
 
 from helm.common.media_object import MediaObject, MultimediaObject
 from helm.benchmark.scenarios.scenario import Input, Instance, Output, Reference
-from .data_augmenter import DataAugmenter
-from .extra_space_perturbation import ExtraSpacePerturbation
-from .misspelling_perturbation import MisspellingPerturbation
-from .contraction_expansion_perturbation import ContractionPerturbation, ExpansionPerturbation
-from .typos_perturbation import TyposPerturbation
-from .filler_words_perturbation import FillerWordsPerturbation
-from .synonym_perturbation import SynonymPerturbation
-from .lowercase_perturbation import LowerCasePerturbation
-from .space_perturbation import SpacePerturbation
-from .dialect_perturbation import DialectPerturbation
-from .person_name_perturbation import PersonNamePerturbation
-from .gender_perturbation import GenderPerturbation
-from .suffix_perturbation import SuffixPerturbation
+from helm.benchmark.augmentations.data_augmenter import DataAugmenter
+from helm.benchmark.augmentations.extra_space_perturbation import ExtraSpacePerturbation
+from helm.benchmark.augmentations.misspelling_perturbation import MisspellingPerturbation
+from helm.benchmark.augmentations.contraction_expansion_perturbation import (
+    ContractionPerturbation,
+    ExpansionPerturbation,
+)
+from helm.benchmark.augmentations.typos_perturbation import TyposPerturbation
+from helm.benchmark.augmentations.filler_words_perturbation import FillerWordsPerturbation
+from helm.benchmark.augmentations.synonym_perturbation import SynonymPerturbation
+from helm.benchmark.augmentations.lowercase_perturbation import LowerCasePerturbation
+from helm.benchmark.augmentations.space_perturbation import SpacePerturbation
+from helm.benchmark.augmentations.dialect_perturbation import DialectPerturbation
+from helm.benchmark.augmentations.person_name_perturbation import PersonNamePerturbation
+from helm.benchmark.augmentations.gender_perturbation import GenderPerturbation
+from helm.benchmark.augmentations.suffix_perturbation import SuffixPerturbation
 
 
 def test_extra_space_perturbation():

helm/benchmark/augmentations/translate_perturbation.py

@@ -2,8 +2,8 @@ from dataclasses import dataclass
 from random import Random
 
 from helm.clients.google_translate_client import GoogleTranslateClient
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class TranslatePerturbation(TextPerturbation):

helm/benchmark/augmentations/typos_perturbation.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from random import Random
 
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
 
 
 class TyposPerturbation(TextPerturbation):

helm/benchmark/data_preprocessor.py

@@ -1,7 +1,7 @@
 from typing import List
 from helm.common.hierarchical_logger import htrack
-from .augmentations.data_augmenter import create_data_augmenter, DataAugmenterSpec, DataAugmenter
-from .scenarios.scenario import Instance, TRAIN_SPLIT, EVAL_SPLITS
+from helm.benchmark.augmentations.data_augmenter import create_data_augmenter, DataAugmenterSpec, DataAugmenter
+from helm.benchmark.scenarios.scenario import Instance, TRAIN_SPLIT, EVAL_SPLITS
 
 
 class DataPreprocessor:

helm/benchmark/huggingface_registration.py

@@ -20,13 +20,10 @@ def register_huggingface_model(
     helm_model_name: str,
     pretrained_model_name_or_path: str,
     revision: Optional[str] = None,
-    openvino: Optional[bool] = False,
 ) -> None:
     object_spec_args: Dict[str, Union[str, bool]] = {"pretrained_model_name_or_path": pretrained_model_name_or_path}
     if revision:
         object_spec_args["revision"] = revision
-    if openvino:
-        object_spec_args["openvino"] = openvino
 
     # Auto-infer model properties from the tokenizer.
     create_tokenizer_args: Dict[str, str] = {"pretrained_model_name_or_path": pretrained_model_name_or_path}
@@ -79,7 +76,7 @@ def register_huggingface_model(
     register_tokenizer_config(tokenizer_config)
 
 
-def register_huggingface_hub_model_from_flag_value(raw_model_string: str, openvino=False) -> None:
+def register_huggingface_hub_model_from_flag_value(raw_model_string: str) -> None:
     raw_model_string_parts = raw_model_string.split("@")
     pretrained_model_name_or_path: str
     revision: Optional[str]
@@ -96,11 +93,10 @@ def register_huggingface_hub_model_from_flag_value(raw_model_string: str, openvi
         helm_model_name=raw_model_string,
         pretrained_model_name_or_path=pretrained_model_name_or_path,
         revision=revision,
-        openvino=openvino,
     )
 
 
-def register_huggingface_local_model_from_flag_value(path: str, openvino=False) -> None:
+def register_huggingface_local_model_from_flag_value(path: str) -> None:
     if not path:
         raise ValueError("Path to Hugging Face model must be non-empty")
     path_parts = os.path.split(path)
@@ -108,5 +104,4 @@ def register_huggingface_local_model_from_flag_value(path: str, openvino=False)
     register_huggingface_model(
         helm_model_name=helm_model_name,
         pretrained_model_name_or_path=path,
-        openvino=openvino,
     )

helm/benchmark/metrics/aci_bench_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.aci_bench_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class ACIBenchMetric(Metric):
+    """Score metrics for ACIBench."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["aci_bench"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("aci_bench_accuracy")).add(score),
+        ]

helm/benchmark/metrics/basic_metrics.py

@@ -5,8 +5,8 @@ from typing import List, Dict, Set
 from urllib.parse import unquote
 
 import numpy as np
-import scipy
-import calibration as cal
+import scipy  # type: ignore
+import calibration as cal  # type: ignore
 from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.metrics.evaluate_reference_metrics import compute_reference_metrics
 from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
@@ -25,10 +25,10 @@ from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
 from helm.benchmark.scenarios.scenario import CORRECT_TAG, Instance
-from .metric import Metric, MetricInterface, MetricResult, add_context, get_unique_stat_by_name
-from .metric_name import MetricContext, MetricName
-from .metric_service import MetricService
-from .statistic import Stat, merge_stat
+from helm.benchmark.metrics.metric import Metric, MetricInterface, MetricResult, add_context, get_unique_stat_by_name
+from helm.benchmark.metrics.metric_name import MetricContext, MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
 
 
 def get_num_bytes(tokens: List[Token]) -> int:

helm/benchmark/metrics/bbq_metrics.py

@@ -3,8 +3,8 @@ from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMe
 
 from helm.common.request import RequestResult
 from helm.benchmark.adaptation.request_state import RequestState
-from .metric_name import MetricName
-from .statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
 
 AMBIGUOUS_TAG = "ambiguous"
 NON_AMBIGUOUS_TAG = "non-ambiguous"

helm/benchmark/metrics/bias_metrics.py

@@ -8,9 +8,18 @@ from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMe
 
 from helm.common.request import RequestResult, GeneratedOutput
 from helm.benchmark.adaptation.request_state import RequestState
-from .statistic import Stat
-from .metric_name import MetricName
-from .bias_word_lists import GENDER_TO_WORD_LISTS, RACE_TO_NAME_LISTS, ADJECTIVE_LIST, PROFESSION_LIST
+from helm.benchmark.metrics.nltk_helper import install_nltk_resources
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.bias_word_lists import (
+    GENDER_TO_WORD_LISTS,
+    RACE_TO_NAME_LISTS,
+    ADJECTIVE_LIST,
+    PROFESSION_LIST,
+)
+
+
+install_nltk_resources()
 
 
 class BiasMetric(EvaluateInstancesMetric):

helm/benchmark/metrics/bigcodebench_metrics.py

@@ -0,0 +1,25 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class BigCodeBenchMetric(Metric):
+    """Score metrics for BigCodeBench."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        score = request_state.annotations["bigcodebench"]["pass_at_one"]
+        return [
+            Stat(MetricName("bigcodebench_p@1")).add(score),
+        ]

helm/benchmark/metrics/bird_sql_metrics.py

@@ -0,0 +1,28 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class BirdSQLMetric(Metric):
+    """Score metrics for Bird-SQL."""
+
+    ANNOTATOR_NAME = "bird_sql"
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        if not request_state.annotations:
+            raise Exception("Request state did not have annotations.")
+        predicted_result = request_state.annotations[self.ANNOTATOR_NAME]["predicted_result"]
+        ground_truth_result = request_state.annotations[self.ANNOTATOR_NAME]["ground_truth_result"]
+        execution_accuracy = int(set(predicted_result) == set(ground_truth_result))
+        return [Stat(MetricName("execution_accuracy")).add(execution_accuracy)]

helm/benchmark/metrics/chw_care_plan_metrics.py

@@ -0,0 +1,34 @@
+from typing import Any, Dict, List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.annotation.chw_care_plan_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class CHWCarePlanMetric(Metric):
+    """Score metrics for CHWCarePla."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        annotations: Dict[str, Any] = request_state.annotations["chw_care_plan"]
+        scores: List[int] = []
+        score = 0.0
+        for annotation_key, annotation_dict in annotations.items():
+            if annotation_key in ANNOTATOR_MODELS.keys() and annotation_dict is not None:
+                for val in annotation_dict.values():
+                    scores.append(int(val["score"]))
+        if scores:
+            score = sum(scores) / len(scores)
+        return [
+            Stat(MetricName("chw_care_plan_accuracy")).add(score),
+        ]

helm/benchmark/metrics/classification_metrics.py

@@ -1,6 +1,6 @@
 from typing import List, Optional
 
-from sklearn.metrics import f1_score
+from sklearn.metrics import f1_score, precision_score, recall_score
 from sklearn.preprocessing import MultiLabelBinarizer
 
 from helm.benchmark.adaptation.request_state import RequestState
@@ -9,9 +9,14 @@ from helm.benchmark.metrics.evaluate_reference_metrics import normalize_text
 from helm.benchmark.metrics.metric import MetricName
 from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.scenarios.scenario import Reference
+from helm.common.hierarchical_logger import hlog
 from helm.common.request import GeneratedOutput
 
 
+def _normalize_label_text(text: str) -> str:
+    return normalize_text(text, should_remove_articles=False)
+
+
 class ClassificationMetric(EvaluateInstancesMetric):
     """Defines metrics for multi-class classification using the generation adapter.
 
@@ -23,8 +28,12 @@ class ClassificationMetric(EvaluateInstancesMetric):
     reference. The predicted class for each instance is the normalized text of the generation.
 
     Note:
-    - The set of classes is derived from the correct references from all the instances.
-      This means that classes may be omitted if they are never used as a correct reference.
+    - It is highly recommended to specify the set of classes should be specified using the
+      `labels` parameter. Otherwise, the set of classes is derived from the correct references
+      from all the instances. This means that classes may be incorrectly omitted if they are never
+      used as a correct reference.
+    - The `averages` parameter is a list of averaging methods to be used.
+      It has the same meaning `average` as in scikit-learn.
     - Generations that are not in any of the known classes are counted as a
       negative prediction for every class.
     - Perturbed classes are considered different classes from unperturbed
@@ -32,8 +41,45 @@ class ClassificationMetric(EvaluateInstancesMetric):
     - Currently, multi-label classification is not supported.
     """
 
-    def __init__(self, delimiter: Optional[str] = None):
+    AVERAGE_OPTIONS = ["micro", "macro", "weighted", None]
+    SCORE_OPTIONS = ["f1", "precision", "recall"]
+
+    def __init__(
+        self,
+        averages: Optional[List[Optional[str]]] = None,
+        labels: Optional[List[str]] = None,
+        scores: Optional[List[str]] = None,
+        delimiter: Optional[str] = None,
+    ) -> None:
+        """Creates metrics for multi-class classification.
+
+        :param delimiter: For multi-label classification, the string delimiter between classes in the model's output.
+        :param average: The list of scores to compute (e.g. "f1", "precision", "recall").
+          Defaults to ["f1"].
+        :param average: The averaging methods (e.g. "micro", "macro", "weighted") to be used.
+          It has the same meaning `average` as in scikit-learn.
+          Defaults to ["macro", "micro"].
+        :param labels: The set of labels.
+        :return: A list of `Stat` objects.
+        """
+        self.averages = averages or ["macro", "micro"]
+        for average in self.averages:
+            if average not in ClassificationMetric.AVERAGE_OPTIONS:
+                raise ValueError(
+                    f"Each value in `averages` must be set to one of {ClassificationMetric.AVERAGE_OPTIONS}."
+                )
+        self.scores = scores or ["f1"]
+        for score_name in self.scores:
+            if score_name not in ClassificationMetric.SCORE_OPTIONS:
+                raise ValueError(f"Each value in `scores` must be set to one of {ClassificationMetric.SCORE_OPTIONS}.")
         self.delimiter = delimiter
+        self.labels = labels
+        if not self.labels:
+            hlog(
+                "WARNING: `labels` were not set on `ClassificationMetric`, "
+                "so they will be inferred from target references. "
+                "It is recommend to explicitly set `labels` on `ClassificationMetric`."
+            )
 
     def is_multi_label(self) -> bool:
         return bool(self.delimiter)
@@ -57,20 +103,38 @@ class ClassificationMetric(EvaluateInstancesMetric):
             references = request_state.instance.all_correct_references
             if not self.is_multi_label():
                 assert len(references) == 1
-            correct_ref_texts = [normalize_text(ref.output.text) for ref in references if ref.output.text]
+            correct_ref_texts = [_normalize_label_text(ref.output.text) for ref in references if ref.output.text]
             y_true.append(correct_ref_texts)
 
             input_text = request_state.result.completions[0].text
             predictions = input_text.split(self.delimiter) if self.is_multi_label() else [input_text]
-            y_pred.append([normalize_text(pred) for pred in predictions if pred])
-        labels: List[str] = list(set(y for ys in y_true for y in ys))
-        mlb = MultiLabelBinarizer().fit([labels])
+            y_pred.append([_normalize_label_text(pred) for pred in predictions if pred])
+        mlb = MultiLabelBinarizer().fit(
+            [[_normalize_label_text(label) for label in self.labels]] if self.labels else y_true
+        )
         y_true = mlb.transform(y_true)
         y_pred = mlb.transform(y_pred)
-        return [
-            Stat(MetricName("classification_macro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="macro")),
-            Stat(MetricName("classification_micro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="micro")),
-        ]
+        stats: List[Stat] = []
+        for average in self.averages:
+            for score_name in self.scores:
+                if score_name == "f1":
+                    score_value = f1_score(y_pred=y_pred, y_true=y_true, average=average)
+                elif score_name == "precision":
+                    score_value = precision_score(y_pred=y_pred, y_true=y_true, average=average)
+                elif score_name == "recall":
+                    score_value = recall_score(y_pred=y_pred, y_true=y_true, average=average)
+                else:
+                    raise ValueError(
+                        f"Unknown score name: '{score_name}' - expected one of ['f1', 'precision', 'recall']"
+                    )
+                if average is None:
+                    for mlb_class, class_score_value in zip(mlb.classes_, score_value):
+                        stats.append(
+                            Stat(MetricName(f"classification_{mlb_class}_{score_name}")).add(class_score_value)
+                        )
+                else:
+                    stats.append(Stat(MetricName(f"classification_{average}_{score_name}")).add(score_value))
+        return stats
 
 
 class MultipleChoiceClassificationMetric(EvaluateInstancesMetric):