crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/benchmark/metrics/evaluate_reference_metrics.py

@@ -0,0 +1,392 @@
+from dataclasses import replace
+from typing import Callable, Dict, List, Optional, Set, Tuple, cast
+import numpy as np
+from functools import partial
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.cleva_metrics_helper import ChineseTokenizer
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.scenarios.code_scenario import CodeReference
+from helm.benchmark.scenarios.scenario import Reference
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.common.request import GeneratedOutput
+from helm.benchmark.scenarios.math_scenario import is_equiv, is_equiv_chain_of_thought
+from nltk.metrics.scores import f_measure
+from nltk.translate.bleu_score import sentence_bleu
+from nltk.tokenize import word_tokenize
+from rouge_score import rouge_scorer
+import re
+import string
+from . import code_metrics_helper
+import nltk
+
+
+try:
+    nltk.data.find("tokenizers/punkt")
+except LookupError:
+    nltk.download("punkt")  # Required for rouge
+
+
+def pass_at_k_estimator(n: int, c: int, k: int) -> float:
+    """Calculates 1 - comb(n - c, k) / comb(n, k).
+
+    Numerically stable version defined in
+        https://arxiv.org/pdf/2107.03374.pdf
+    """
+    if n - c < k:
+        return 1.0
+    return 1.0 - np.prod(1.0 - k / np.arange(n - c + 1, n + 1))
+
+
+def normalize_text(text: str) -> str:
+    """Lower text and remove punctuation, articles and extra whitespace.
+    Copied from the [QuAC](http://quac.ai/) evaluation script found at
+    https://s3.amazonaws.com/my89public/quac/scorer.py"""
+
+    def remove_articles(text: str) -> str:
+        return re.sub(r"\b(a|an|the)\b", " ", text)
+
+    def white_space_fix(text: str) -> str:
+        return " ".join(text.split())
+
+    def remove_punc(text: str) -> str:
+        exclude = set(string.punctuation)
+        return "".join(ch for ch in text if ch not in exclude)
+
+    def lower(text: str) -> str:
+        return text.lower()
+
+    return white_space_fix(remove_articles(remove_punc(lower(text))))
+
+
+def exact_match(gold: str, pred: str) -> float:
+    if not pred:
+        return 0
+
+    return 1 if gold.strip() == pred.strip() else 0
+
+
+def quasi_exact_match(gold: str, pred: str) -> float:
+    if not pred:
+        return 0
+
+    return 1 if normalize_text(gold) == normalize_text(pred) else 0
+
+
+def prefix_exact_match(gold: str, pred: str) -> float:
+    """
+    The `prefix_exact_match` metric is particularly useful in the zero-shot setting, where the model is
+    not given examples of the expected outputs and tends to output more tokens than it should.
+
+    For example, for this zero-shot prompt from BoolQ,
+
+    Passage: Elmendorf Air Force Base (IATA: EDF, ICAO: PAED, FAA LID: EDF) is a United States military facility
+    in Anchorage, the largest city in Alaska. Originally known as Elmendorf Field, it became Elmendorf Air Force
+    Base after World War II, and in 2010 it merged with nearby Fort Richardson to form Joint Base Elmendorf-Richardson.
+    Question: Is there an air force base in anchorage alaska?
+    Answer:
+
+    the model could output up to `max_tokens` number of tokens "Yes, Elmendorf" instead of just "Yes".
+    """
+    if not pred:
+        return 0
+
+    return 1 if pred.strip().startswith(gold.strip()) else 0
+
+
+def quasi_prefix_exact_match(gold: str, pred: str) -> float:
+    """
+    Same thing as `prefix_exact_match` but we normalize the text before checking if the prefix match.
+    """
+    if not pred:
+        return 0
+
+    return 1 if normalize_text(pred).startswith(normalize_text(gold)) else 0
+
+
+def f1_score(gold: str, pred: str) -> float:
+    ret = f_measure(set(normalize_text(gold).split()), set(normalize_text(pred).split()))
+    if ret is None:  # answer is the empty string after normalizing
+        return 0.0
+
+    return ret
+
+
+def exact_match_indicator(gold: str, pred: str, indicator: str = " ") -> float:
+    """
+    Exact match, allowing for some preceding context.
+    For example, the following two answers are considered matching:
+    - Because of x and y, the answer is ## <answer>
+    - Given reasons y and z, the answer is ## <answer>
+    While the following is considered different from the earlier two
+    - Given reasons x and a, the answer is ## <other answer>
+    """
+    pred = pred.split(indicator)[-1].strip()
+    gold = gold.split(indicator)[-1].strip()
+    return exact_match(gold, pred)
+
+
+def final_number_exact_match(gold: str, pred: str) -> float:
+    """
+    Returns 1 iff the final number in gold and pred match.
+    Similar to exact_match_indicator.
+    Example:
+    - gold = "The answer is 15."
+    - pred = "The answer is 15 eggs."
+    - Returns 1
+    """
+
+    def get_final_number(x: str) -> str:
+        matches = re.findall(r"-?[\d,]+(?:.\d+)?", x)
+        if not matches:
+            return ""
+        return matches[-1].replace(",", "")
+
+    return exact_match(get_final_number(gold), get_final_number(pred))
+
+
+def rouge_score(gold: str, pred: str, rouge_type: str, scorer: rouge_scorer.RougeScorer) -> float:
+    scores = scorer.score(gold, pred)
+    return scores[rouge_type].fmeasure
+
+
+def get_rouge_function(rouge_type: str) -> Callable[[str, str], float]:
+    scorer = rouge_scorer.RougeScorer([rouge_type], use_stemmer=True)
+    return partial(rouge_score, scorer=scorer, rouge_type=rouge_type)
+
+
+def bleu_1(gold: str, pred: str) -> float:
+    return sentence_bleu([word_tokenize(gold)], word_tokenize(pred), weights=(1, 0, 0, 0))
+
+
+def chinese_bleu_1(gold: str, pred: str) -> float:
+    char_tokenizer = ChineseTokenizer()
+    return sentence_bleu([char_tokenizer.tokenize(gold)], char_tokenizer.tokenize(pred), weights=(1, 0, 0, 0))
+
+
+def get_chinese_rouge_function(rouge_type: str) -> Callable[[str, str], float]:
+    char_tokenizer = ChineseTokenizer()
+    scorer = rouge_scorer.RougeScorer([rouge_type], use_stemmer=True, tokenizer=char_tokenizer)
+    return partial(rouge_score, scorer=scorer, rouge_type=rouge_type)
+
+
+def cleva_math_result_match(gold: str, pred: str) -> float:
+    """
+    Exact match that only cares the last math expression.
+    Common math expressions are numbers and fractions.
+    """
+    pattern = r"[-+*/%\.\(\)\d]+"
+    matches = re.findall(pattern, pred)
+    if matches:
+        pred = matches[-1].lstrip(")")
+    # remove space in front or at the end
+    pred = pred.strip()
+    return exact_match(gold, pred)
+
+
+def bleu_4(gold: str, pred: str) -> float:
+    return sentence_bleu([word_tokenize(gold)], word_tokenize(pred), weights=(0, 0, 0, 1))
+
+
+def cider(gold: str, pred: str) -> float:
+    try:
+        from pycocoevalcap.cider.cider import Cider
+    except ModuleNotFoundError as e:
+        handle_module_not_found_error(e, ["vlm"])
+
+    cider_evaluator = Cider()
+    candidate = {"caption": [pred]}
+    reference = {"caption": [gold]}
+    average_score, _ = cider_evaluator.compute_score(reference, candidate)
+    return average_score
+
+
+def extract_set_from_text(
+    set_str: str,
+    set_start_str: str = " is ",
+    set_separator: str = " and ",
+    empty_set_str: str = "Nothing.",
+) -> Set[str]:
+    """
+    Given a string, extract the set of strings implied by that string.
+    set_start_str denotes the start of the set
+    set_separator denotes the string separating set elements
+    empty_set_str is the string which denotes the empty set
+    """
+    if set_str == empty_set_str:
+        return set()
+    set_str = set_str.replace(".", "")
+    extracted_set = set(set_str.split(set_start_str)[-1].split(set_separator))
+    return extracted_set
+
+
+def extract_gold_pred_sets(gold: str, pred: str) -> Tuple[Set[str], Set[str]]:
+    """Extract the set of strings implied by the gold and pred strings"""
+    gold_set = extract_set_from_text(gold)
+    pred_set = extract_set_from_text(pred.split("\n")[0])
+    return gold_set, pred_set
+
+
+def iou_set_match(gold: str, pred: str) -> float:
+    """Compute the intersection over union of the gold and pred sets"""
+    gold_set, pred_set = extract_gold_pred_sets(gold, pred)
+    if len(gold_set) == 0:  # If gold is empty, just check if the pred set is also empty
+        return float(gold_set == pred_set)
+    return len(gold_set.intersection(pred_set)) / len(gold_set.union(pred_set))
+
+
+def f1_set_match(gold: str, pred: str) -> float:
+    """Compute the F1 score of the gold and pred sets"""
+    gold_set, pred_set = extract_gold_pred_sets(gold, pred)
+    if len(gold_set) == 0:  # If gold is empty, just check if the pred set is also empty
+        return float(gold_set == pred_set)
+    true_positives = gold_set.intersection(pred_set)
+    return 2 * len(true_positives) / (len(gold_set) + len(pred_set))
+
+
+def exact_set_match(gold: str, pred: str) -> float:
+    """Compute whether the sets generated exactly match"""
+    gold_set, pred_set = extract_gold_pred_sets(gold, pred)
+    return float(gold_set == pred_set)
+
+
+def absolute_value_difference(gold: str, pred: str) -> float:
+    """Compute the absolute value of the difference between two numbers (provided as strings),
+    or 0.0 if invalid input.
+    """
+
+    def maybe_int(text: str):
+        """Parse int, ignoring commas in numbers."""
+        try:
+            val = int(text.replace(",", ""))
+        except ValueError:
+            return 0.0
+        return val
+
+    gold_val = maybe_int(gold)
+    pred_val = maybe_int(pred)
+    return abs(gold_val - pred_val)
+
+
+def code_eval(gold: Tuple[str, Optional[Dict]], pred: str) -> float:
+    """Evaluate Code Correctness on test examples."""
+    assert gold[1] is not None  # gold[1]["canonical_solution"]
+    # Warning: will execute machine generated code; need to sandbox before executing
+    return float(code_metrics_helper.check_correctness(gold[1], pred, 3.0)["passed"])  # type: ignore
+
+
+# TODO This should probably be made into an implementation of MetricInterface. For now it lives here
+# just to separate it from basic_metrics.py.
+def compute_reference_metrics(
+    names: List[str], adapter_spec: AdapterSpec, request_state: RequestState, metric_service: MetricService
+) -> List[Stat]:
+    """
+    Setup:
+
+    - Gold (correct references): G1 ... Gm
+    - Predictions (completions): P1 ... Pk
+
+    For each pair (G, P), we can define a ${score} (e.g., exact match, F1, BLEU).
+
+    We define the following stats:
+
+    - ${score}: max_i score(Gi, P1)
+    - ${score}@k: max_{i,j} score(Gi, Pj)
+    """
+
+    def compute_metrics_helper(
+        name: MetricName,
+        score_func: Callable,
+        group: Optional[str] = None,
+    ) -> List[Stat]:
+        if name.name == "pass":  # Calculate pass@k for HumanEval from CodeScenario.
+            score_func = cast(Callable[[Tuple[str, Optional[Dict]], str], float], score_func)  # Make mypy happy.
+            code_golds = cast(List[CodeReference], golds)
+            results = [score_func((gold.output.text, gold.test_cases), pred) for gold in code_golds for pred in preds]
+            _len, _sum = len(results), int(sum(results))  # Cast to int to make type match.
+            score_1 = pass_at_k_estimator(_len, _sum, 1)
+            score_k = pass_at_k_estimator(_len, _sum, adapter_spec.num_outputs)
+        elif name.name == "code_eval_acc":
+            score_func = cast(Callable[[Tuple[str, Optional[Dict]], str], float], score_func)  # Make mypy happy.
+            code_golds = cast(List[CodeReference], golds)
+            score_1 = max(score_func((gold.output.text, gold.test_cases), preds[0]) for gold in code_golds)
+            score_k = max(
+                score_func((gold.output.text, gold.test_cases), pred) for gold in code_golds for pred in preds
+            )
+        else:
+            score_func = cast(Callable[[str, str], float], score_func)  # Make mypy happy.
+            score_1 = max(score_func(gold.output.text, preds[0]) for gold in golds)
+            score_k = max(score_func(gold.output.text, pred) for gold in golds for pred in preds)
+
+        metrics = [Stat(name).add(score_1)]  # score_1 corresponds using one prediction
+        if adapter_spec.num_outputs != 1:
+            metrics.append(Stat(replace(name, name=f"{name.name}@{adapter_spec.num_outputs}")).add(score_k))
+        return metrics
+
+    # maps each string metric name to its associated function
+    metric_fn_mapping: Dict[str, Callable] = {
+        "exact_match": exact_match,
+        "quasi_exact_match": quasi_exact_match,
+        "prefix_exact_match": prefix_exact_match,
+        "quasi_prefix_exact_match": quasi_prefix_exact_match,
+        "exact_match_indicator": exact_match_indicator,
+        "final_number_exact_match": final_number_exact_match,
+        "exact_set_match": exact_set_match,
+        "iou_set_match": iou_set_match,
+        "f1_set_match": f1_set_match,
+        "math_equiv": is_equiv,
+        "math_equiv_chain_of_thought": is_equiv_chain_of_thought,
+        "code_eval_acc": code_eval,
+        "pass": code_eval,
+        "cider": cider,
+        "f1_score": f1_score,
+        "rouge_1": get_rouge_function("rouge1"),
+        "rouge_2": get_rouge_function("rouge2"),
+        "rouge_l": get_rouge_function("rougeL"),
+        "bleu_1": bleu_1,
+        "bleu_4": bleu_4,
+        "chinese_bleu_1": chinese_bleu_1,
+        "chinese_rouge_1": get_chinese_rouge_function("rouge1"),
+        "chinese_rouge_2": get_chinese_rouge_function("rouge2"),
+        "cleva_math_result_match": cleva_math_result_match,
+        "absolute_value_difference": absolute_value_difference,
+    }
+
+    stats: List[Stat] = []
+
+    # Gold outputs
+    golds: List[Reference] = [reference for reference in request_state.instance.references if reference.is_correct]
+    assert len(golds) > 0
+
+    # Predicted outputs
+    assert request_state.result is not None
+    sorted_completions: List[GeneratedOutput] = sorted(request_state.result.completions, key=lambda x: -x.logprob)
+    preds: List[str] = [completion.text.strip() for completion in sorted_completions]
+
+    # Apply mapping if exists (e.g., for multiple-choice questions A -> Boston, B -> New York)
+    # Note: If 'A' and 'B' were the only possible choices, smaller language models like GPT-2 would
+    # sometimes predict a random letter like 'M'.
+    if request_state.output_mapping is not None:
+        preds = [request_state.output_mapping.get(pred) for pred in preds]  # type: ignore
+
+    # Compute max_prob, the probability that the model assigns to its generated text.
+    # Use the log prob of sorted_completions[0], which is the completion with the highest
+    # log_prob. We use this since that's what's used for computing metrics like exact_match.
+    # One subtlety is that when computing exact_match, we strip whitespace, so the actual
+    # max_prob is the sum of all the probabilities in the set {x : strip(x) = prediction}.
+    # In practice, we think this may not make much of a difference because models may not place
+    # high probabilities on having additional spaces (should check this). Also, the sum
+    # involves computing the log_prob for many completions which could be intractable.
+    max_prob = np.exp(sorted_completions[0].logprob)
+    stats.append(Stat(MetricName("max_prob")).add(max_prob))
+
+    # Add other metrics
+    for metric_name in names:
+        if metric_name in metric_fn_mapping:
+            stats.extend(compute_metrics_helper(MetricName(metric_name), metric_fn_mapping[metric_name]))
+        else:
+            raise NameError(f"{metric_name} is not in the list of metric functions.")
+
+    return stats

helm/benchmark/metrics/image_generation/aesthetics_metrics.py

@@ -0,0 +1,54 @@
+from statistics import mean
+from typing import List, Optional
+
+from helm.common.images_utils import is_blacked_out_image
+from helm.common.request import RequestResult
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from .aesthetics_scorer import AestheticsScorer
+from helm.common.multimodal_request_utils import gather_generated_image_locations
+
+
+class AestheticsMetric(Metric):
+    """
+    Defines metrics for LAION's CLIP-based aesthetics predictor for images
+    (https://github.com/LAION-AI/aesthetic-predictor).
+    """
+
+    def __init__(self):
+        self._aesthetics_scorer: Optional[AestheticsScorer] = None
+
+    def __repr__(self):
+        return "AestheticsMetric()"
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.result is not None
+        request_result: RequestResult = request_state.result
+        image_locations: List[str] = gather_generated_image_locations(request_result)
+        if len(image_locations) == 0:
+            return []
+
+        if self._aesthetics_scorer is None:
+            self._aesthetics_scorer = AestheticsScorer()
+
+        # Compute the aesthetics score for each generated image. Skip blacked out images.
+        scores: List[float] = [
+            self._aesthetics_scorer.compute_aesthetics_score(location)
+            for location in image_locations
+            if not is_blacked_out_image(location)
+        ]
+        stats: List[Stat] = [
+            Stat(MetricName("expected_aesthetics_score")).add(mean(scores) if len(scores) > 0 else 0),
+            Stat(MetricName("max_aesthetics_score")).add(max(scores) if len(scores) > 0 else 0),
+        ]
+        return stats

helm/benchmark/metrics/image_generation/aesthetics_scorer.py

@@ -0,0 +1,66 @@
+from urllib.request import urlretrieve
+import os
+
+import torch
+
+from helm.common.general import ensure_directory_exists
+from helm.common.gpu_utils import get_torch_device
+from helm.common.images_utils import open_image
+from helm.common.optional_dependencies import handle_module_not_found_error
+from helm.benchmark.runner import get_cached_models_path
+
+
+class AestheticsScorer:
+    """
+    LAION's CLIP-based aesthetics predictor for images (https://github.com/LAION-AI/aesthetic-predictor).
+    Adapted from
+    https://colab.research.google.com/github/LAION-AI/aesthetic-predictor/blob/main/asthetics_predictor.ipynb.
+    """
+
+    MODEL_URL_TEMPLATE: str = (
+        "https://github.com/LAION-AI/aesthetic-predictor/blob/main/sa_0_4_{clip_model}_linear.pth?raw=true"
+    )
+
+    @staticmethod
+    def load_model(clip_model="vit_l_14"):
+        """Load the aesthetics model."""
+        cache_folder: str = os.path.join(get_cached_models_path(), "emb_reader")
+        ensure_directory_exists(cache_folder)
+        model_path: str = os.path.join(cache_folder, f"sa_0_4_{clip_model}_linear.pth")
+
+        if not os.path.exists(model_path):
+            model_url: str = os.path.join(AestheticsScorer.MODEL_URL_TEMPLATE.format(clip_model=clip_model))
+            urlretrieve(model_url, model_path)
+
+        if clip_model == "vit_l_14":
+            m = torch.nn.Linear(768, 1)
+        elif clip_model == "vit_b_32":
+            m = torch.nn.Linear(512, 1)
+        else:
+            raise ValueError(f"Invalid model: {clip_model}")
+
+        s = torch.load(model_path)
+        m.load_state_dict(s)
+        m.eval()
+        return m
+
+    def __init__(self):
+        try:
+            import clip
+        except ModuleNotFoundError as e:
+            handle_module_not_found_error(e, ["heim"])
+
+        # Load the CLIP and aesthetics model
+        self._device: torch.device = get_torch_device()
+        self._model, self._preprocess = clip.load("ViT-L/14", device=self._device)
+        self._aesthetics_model = self.load_model().to(self._device)
+
+    def compute_aesthetics_score(self, image_location: str) -> float:
+        """
+        Compute the aesthetics score. Returns a value between 1 and 10.
+        """
+        image = self._preprocess(open_image(image_location)).unsqueeze(0).to(self._device)
+        with torch.no_grad():
+            image_features = self._model.encode_image(image)
+            image_features /= image_features.norm(dim=-1, keepdim=True)
+            return self._aesthetics_model(image_features.float()).detach().item()

helm/benchmark/metrics/image_generation/clip_score_metrics.py

@@ -0,0 +1,73 @@
+from statistics import mean
+from typing import List
+
+from helm.common.general import singleton
+from helm.common.request import RequestResult
+from helm.common.clip_score_request import DEFAULT_CLIP_SCORE_MODEL, CLIPScoreResult, CLIPScoreRequest
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
+from helm.common.images_utils import is_blacked_out_image
+from helm.common.multimodal_request_utils import gather_generated_image_locations
+
+
+class CLIPScoreMetric(Metric):
+    """
+    Defines CLIPScore-based metrics (https://arxiv.org/abs/2104.08718).
+    CLIPScore is a reference free metric that can be used to evaluate the correlation between an image
+    caption and the content of the image. It has been found to be highly correlated with human judgement.
+    """
+
+    def __init__(self, multilingual: bool = False):
+        self._multilingual: bool = multilingual
+
+    def __repr__(self):
+        return f"CLIPScoreMetric(multilingual={self._multilingual})"
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        def get_metric_name(base_name: str) -> str:
+            if self._multilingual:
+                base_name = f"{base_name}_multilingual"
+            return base_name
+
+        assert request_state.result is not None
+        request_result: RequestResult = request_state.result
+
+        prompt: str = request_state.request.prompt
+        perturbation_name: str = request_state.instance.perturbation.name if request_state.instance.perturbation else ""
+        if (
+            request_state.instance.contrast_inputs is not None
+            and len(request_state.instance.contrast_inputs) > 0
+            and perturbation_name in ["translate", "dialect", "mild_mix"]
+        ):
+            prompt = singleton(request_state.instance.contrast_inputs).text
+
+        # Truncate the prompt using the CLIP tokenizer before feeding into the CLIP model.
+        # Otherwise, the library will throw an error.
+        model = DEFAULT_CLIP_SCORE_MODEL
+        prompt = WindowServiceFactory.get_window_service(model, metric_service).truncate_from_right(prompt)
+
+        scores: List[float] = []
+        image_locations: List[str] = gather_generated_image_locations(request_result)
+        for location in image_locations:
+            if not is_blacked_out_image(location):
+                result: CLIPScoreResult = metric_service.compute_clip_score(
+                    CLIPScoreRequest(prompt, location, model=model, multilingual=self._multilingual)
+                )
+                scores.append(result.score)
+
+        stats: List[Stat] = [
+            Stat(MetricName(get_metric_name("expected_clip_score"))).add(mean(scores) if len(scores) > 0 else 0),
+            Stat(MetricName(get_metric_name("max_clip_score"))).add(max(scores) if len(scores) > 0 else 0),
+        ]
+        return stats

helm/benchmark/metrics/image_generation/denoised_runtime_metric.py

@@ -0,0 +1,42 @@
+from collections import defaultdict
+from tqdm import tqdm
+from typing import Dict
+import math
+import numpy as np
+
+from helm.common.request import RequestResult
+from helm.benchmark.scenarios.scenario import Instance
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+
+
+class DenoisedRuntimeMetric(MetricInterface):
+    def __repr__(self):
+        return "DenoisedRuntimeMetric()"
+
+    def evaluate(
+        self,
+        scenario_state: ScenarioState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+        parallelism: int,
+    ) -> MetricResult:
+
+        instance_to_min_request_times: Dict[Instance, float] = defaultdict(lambda: math.inf)
+        for request_state in tqdm(scenario_state.request_states):
+            assert request_state.result is not None
+            request_result: RequestResult = request_state.result
+
+            assert request_result.request_time is not None
+            request_time: float = request_result.request_time
+
+            instance: Instance = request_state.instance
+            instance_to_min_request_times[instance] = min(instance_to_min_request_times[instance], request_time)
+
+        denoised_runtime: float = float(np.mean(list(instance_to_min_request_times.values())))
+        return MetricResult(
+            aggregated_stats=[Stat(MetricName("denoised_runtime")).add(denoised_runtime)], per_instance_stats=[]
+        )

helm/benchmark/metrics/image_generation/detection_metrics.py

@@ -0,0 +1,57 @@
+from typing import List, Dict, Any
+import json
+from statistics import mean
+
+from helm.common.request import RequestResult
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.common.multimodal_request_utils import gather_generated_image_locations
+from .detectors.vitdet import ViTDetDetector
+
+
+class DetectionMetric(Metric):
+    """
+    Define metrics following DALL-EVAL (https://arxiv.org/abs/2202.04053),
+    which measure whether generated images contain the correct objects, counts, and relations
+    as specified in input text prompts.
+    """
+
+    def __init__(self):
+        self._detection_model = None
+
+    def __repr__(self):
+        return "DetectionMetric()"
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.result is not None
+        request_result: RequestResult = request_state.result
+        image_locations: List[str] = gather_generated_image_locations(request_result)
+        if len(image_locations) == 0:
+            return []
+
+        if self._detection_model is None:
+            self._detection_model = ViTDetDetector()
+
+        instance = request_state.instance
+        references: Dict[str, Any] = {**json.loads(instance.references[0].output.text), "skill": instance.sub_split}
+
+        prompt: str = request_state.request.prompt
+        scores: List[float] = []
+        for image_location in image_locations:
+            score: float = self._detection_model.compute_score(prompt, image_location, references)
+            scores.append(score)
+
+        stats: List[Stat] = [
+            Stat(MetricName("detection_correct_frac")).add(mean(scores) if len(scores) > 0 else 0),
+        ]
+        return stats