crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/metrics/metric_service.py

@@ -2,9 +2,13 @@ from typing import Optional
 
 from helm.common.authentication import Authentication
 from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
+from helm.common.file_upload_request import FileUploadResult, FileUploadRequest
+from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
+from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
 from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
 from helm.proxy.services.service import Service
+from helm.common.cache import Cache
 
 
 class MetricService(TokenizerService):
@@ -15,8 +19,20 @@ class MetricService(TokenizerService):
     def __init__(self, service: Service, auth: Authentication):
         super().__init__(service, auth)
 
+    def check_nudity(self, request: NudityCheckRequest) -> NudityCheckResult:
+        return self._service.check_nudity(self._auth, request)
+
+    def compute_clip_score(self, request: CLIPScoreRequest) -> CLIPScoreResult:
+        return self._service.compute_clip_score(self._auth, request)
+
+    def upload(self, request: FileUploadRequest) -> FileUploadResult:
+        return self._service.upload(self._auth, request)
+
     def get_toxicity_scores(self, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
         return self._service.get_toxicity_scores(self._auth, request)
 
     def make_critique_request(self, request: CritiqueRequest) -> Optional[CritiqueRequestResult]:
         return self._service.make_critique_request(self._auth, request)
+
+    def get_cache(self, shard_name: str) -> Cache:
+        return Cache(self._service.get_cache_config(shard_name))

helm/benchmark/metrics/paraphrase_generation_metrics.py

@@ -1,13 +1,13 @@
 from typing import List
 
 from helm.benchmark.adaptation.request_state import RequestState
-from .metric import Metric
+from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
 from .metric_name import MetricName
 from .statistic import Stat
 from nltk.translate.bleu_score import corpus_bleu
 
 
-class CLEVAParaphraseGenerationMetric(Metric):
+class CLEVAParaphraseGenerationMetric(EvaluateInstancesMetric):
     """
     Compute the Chinese iBLEU score for Paraphrase Generation scenarios of CLEVA benchmark.
     This implementation allows variable number of references (i.e., golds).
@@ -21,8 +21,7 @@ class CLEVAParaphraseGenerationMetric(Metric):
     def __init__(self, alpha: float = 0.8):  # calculate iBLEU_0.8 by default
         self.alpha = alpha
 
-    def evaluate_instances(self, request_states: List[RequestState]) -> List[Stat]:
-
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
         inputs: List = []
         preds: List = []
         golds: List[List[str]] = []

helm/benchmark/metrics/ranking_metrics.py

@@ -4,11 +4,11 @@ from typing import Callable, Dict, List, Tuple, Optional
 from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_RANKING_BINARY
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.reference_metric import ReferenceMetric
 from helm.common.optional_dependencies import handle_module_not_found_error
 from helm.benchmark.scenarios.scenario import unpack_tag, CORRECT_TAG, Reference
 from helm.common.request import RequestResult
-from helm.common.general import binarize_dict
-from .metric import Metric
+from helm.common.general import assert_present, binarize_dict
 from .metric_name import MetricName
 from .metric_service import MetricService
 from .statistic import Stat
@@ -58,7 +58,7 @@ class RankingObject:
     model_relevance: Optional[int] = None
 
 
-class RankingMetric(Metric):
+class RankingMetric(ReferenceMetric):
     """Ranking metric."""
 
     """ Methods supported by this metric.
@@ -205,14 +205,13 @@ class RankingMetric(Metric):
         relevance dictionary, which contains the ground truth relevance
         values for each document.
         """
-        assert all([r.model_relevance is not None for r in ranking_objs])
         if rank_limit:
             return {
-                self.get_query_string(r.reference_index): r.model_relevance  # type: ignore
+                self.get_query_string(r.reference_index): assert_present(r.model_relevance)
                 for r in ranking_objs
                 if r.rank and r.rank <= rank_limit
             }
-        return {self.get_query_string(r.reference_index): r.model_relevance for r in ranking_objs}  # type: ignore
+        return {self.get_query_string(r.reference_index): assert_present(r.model_relevance) for r in ranking_objs}
 
     def get_true_relevances(self, ranking_objects: List[RankingObject]) -> Dict[str, int]:
         """Get the true relevance dictionary."""
@@ -372,7 +371,7 @@ class RankingMetric(Metric):
         #   len(ranking_objects) minus its relevance.
         stats += [
             Stat(MetricName(f"ref{r.reference_index}_rank")).add(
-                len(ranking_objects) - r.model_relevance  # type: ignore
+                len(ranking_objects) - assert_present(r.model_relevance)
             )
             for r in ranking_objects
         ]

helm/benchmark/metrics/reference_metric.py

@@ -0,0 +1,148 @@
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+from typing import List, Dict
+from helm.benchmark.metrics.metric import (
+    MetricInterface,
+    MetricResult,
+    PerInstanceStats,
+    add_context,
+    compute_worst_case_metrics,
+)
+
+from helm.common.general import parallel_map
+from helm.benchmark.adaptation.adapters.adapter_factory import ADAPT_LANGUAGE_MODELING
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from .metric_name import MetricName, MetricContext
+from .metric_service import MetricService
+from .statistic import Stat, merge_stat
+
+
+@dataclass(frozen=True)
+class Processor:
+    """Evaluates an instance."""
+
+    # TODO: not ideal that we have circular dependencies; subclasses of Metric
+    # should override the Processor rather than the Metric.
+    metric: "ReferenceMetric"
+    metric_service: MetricService
+    eval_cache_path: str
+    adapter_spec: AdapterSpec
+
+    def process(self, references_states: List[RequestState]) -> List[Stat]:
+        instance_stats: List[Stat] = []
+
+        # Evaluate the references
+        if len(references_states) == 0:
+            return instance_stats
+        instance_stats.extend(
+            self.metric.evaluate_references(
+                self.adapter_spec, references_states, self.metric_service, self.eval_cache_path
+            )
+        )
+
+        # Add instance-related context (e.g., split, perturbation) to the metrics
+        for i, stat in enumerate(instance_stats):
+            instance_stats[i] = add_context(stat, MetricContext.from_instance(references_states[0].instance))
+
+        return instance_stats
+
+
+class ReferenceMetric(MetricInterface, ABC):
+    """
+    A `Metric` takes the results of execution and produces `Stat`s for a
+    scenario.
+
+    Note: `Metric` actually right now is a bit of misnomer because it produces many
+    `Stat`s, that might be distinct but are computed together.  Eventually we
+    might move to a world where there is one (or very few metrics that are domain-independent).
+    """
+
+    def evaluate(
+        self, scenario_state: ScenarioState, metric_service: MetricService, eval_cache_path: str, parallelism: int
+    ) -> MetricResult:
+        """
+        Main entry point for a `Metric`.  This function groups the single
+        list of `RequestState` by training trial and instance, and invokes
+        other functions to process those.  This should serve most purposes.
+
+        Any logic that doesn't decompose along instances should go here, such
+        as robustness.
+        """
+        assert scenario_state.adapter_spec.method != ADAPT_LANGUAGE_MODELING, (
+            "Metric no longer knows how to handle the language modeling adapter. "
+            + "All run_specs with that adapter should use LanguageModelingMetric. "
+            + "If you are seeing this issue, please file a Github issue."
+        )
+
+        adapter_spec = scenario_state.adapter_spec
+        global_stats: Dict[MetricName, Stat] = {}
+        all_per_instance_stats: List[PerInstanceStats] = []
+
+        for train_trial_index in range(adapter_spec.num_train_trials):
+            # Construct inputs
+            request_state_sets: List[List[RequestState]] = []
+            for instance in scenario_state.instances:
+                references_states = []
+                for reference_index in range(len(instance.references)):
+                    references_states.extend(
+                        scenario_state.get_request_states(train_trial_index, instance, reference_index)
+                    )
+                request_state_sets.append(references_states)
+
+            # Do it!
+            processor = Processor(
+                metric=self,
+                metric_service=metric_service,
+                eval_cache_path=eval_cache_path,
+                adapter_spec=scenario_state.adapter_spec,
+            )
+            results: List[List[Stat]] = parallel_map(
+                processor.process,
+                request_state_sets,
+                parallelism=parallelism,
+            )
+
+            # Compute per-instance stats
+            per_instance_stats: List[PerInstanceStats] = []
+            for instance, stats in zip(scenario_state.instances, results):
+                assert instance.id is not None, f"id was none for instance: {instance}"
+                # Sometimes a metric (e.g., BiasMetric) doesn't produce any statistics
+                if len(stats) > 0:
+                    per_instance_stats.append(
+                        PerInstanceStats(instance.id, instance.perturbation, train_trial_index, stats)
+                    )
+
+            # Aggregate these stats
+            trial_stats: Dict[MetricName, Stat] = {}  # Statistics just for this trial
+            for instance_stats in results:
+                for stat in instance_stats:
+                    merge_stat(trial_stats, stat)
+
+            # Compute worst-case metrics.
+            # This is here since we want these stats for all metrics and they
+            # aggregate across contexts (perturbations).
+            worst_case_stats = compute_worst_case_metrics(dict(zip(scenario_state.instances, results)))
+            for stat in worst_case_stats:
+                merge_stat(trial_stats, stat)
+
+            # We take the mean value for each trial.
+            for stat in trial_stats.values():
+                merge_stat(global_stats, stat.take_mean())
+
+            all_per_instance_stats.extend(per_instance_stats)
+
+        # Wrap aggregated and per-instance stats in a MetricResult.
+        return MetricResult(list(global_stats.values()), all_per_instance_stats)
+
+    @abstractmethod
+    def evaluate_references(
+        self,
+        adapter_spec: AdapterSpec,
+        reference_request_states: List[RequestState],
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        """Evaluate the references.  Override me!"""
+        pass

helm/benchmark/metrics/summac/model_summac.py

@@ -50,7 +50,6 @@ class SummaCImager:
     def __init__(
         self, model_name="mnli", granularity="paragraph", use_cache=True, max_doc_sents=100, device="cuda", **kwargs
     ):
-
         self.grans = granularity.split("-")
 
         assert (
@@ -154,7 +153,6 @@ class SummaCImager:
             for j in range(N_gen)
         ]
         for batch in utils_misc.batcher(dataset, batch_size=20):
-
             if self.model_name == "decomp":
                 batch_evids, batch_conts, batch_neuts = [], [], []
                 batch_json = [{"premise": d["premise"], "hypothesis": d["hypothesis"]} for d in batch]

helm/benchmark/metrics/summarization_metrics.py

@@ -5,19 +5,20 @@ import os
 import pickle
 
 import spacy
+import spacy.cli
 from typing import List, Dict, Optional
 from collections import defaultdict
 
 from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.metrics.evaluate_reference_metrics import get_rouge_function
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
 from helm.common.optional_dependencies import handle_module_not_found_error
 from .metric import Metric, MetricResult
 from .metric_name import MetricName
 from .metric_service import MetricService
-from .basic_metrics import get_rouge_function
 from .statistic import Stat
 from .summac.model_summac import SummaCZS
 from bert_score import BERTScorer
@@ -52,12 +53,12 @@ class SummarizationMetric(Metric):
         # avoid triggering a bug in DataStatsMetric that raises
         # `NameError: name 'stderr' is not defined`
         if not spacy.util.is_package("en_core_web_sm"):
-            spacy.cli.download("en_core_web_sm")  # type: ignore
+            spacy.cli.download("en_core_web_sm")
 
         try:
             from summ_eval.data_stats_metric import DataStatsMetric
         except ModuleNotFoundError as e:
-            handle_module_not_found_error(e, ["metrics"])
+            handle_module_not_found_error(e, ["summarization"])
 
         self.data_stats_metric = DataStatsMetric()
         self.task: str = task
@@ -168,7 +169,6 @@ class SummarizationMetric(Metric):
         metric_service: MetricService,
         eval_cache_path: str,
     ) -> List[Stat]:
-
         refs: List[str] = [self._remove_braces(ref.output.text) for ref in request_state.instance.references]
         inp: str = self._remove_braces(request_state.instance.input.text)
 
@@ -182,9 +182,9 @@ class SummarizationMetric(Metric):
                 self.humaneval = self._load_humaneval(eval_cache_path)
 
             # get human evaluation scores if they exist
-            model_name = adapter_spec.model.replace("/", "_")
+            deployment = adapter_spec.model_deployment.replace("/", "_")
             for metric_name in ["faithfulness", "relevance", "coherence"]:
-                val = self.humaneval[(metric_name, model_name, request_state.instance.id, pred)]
+                val = self.humaneval[(metric_name, deployment, request_state.instance.id, pred)]
                 result.append(Stat(MetricName(f"HumanEval-{metric_name}")).add(float(val)))
         except KeyError:
             pass
@@ -196,8 +196,8 @@ class SummarizationMetric(Metric):
             if self.qa_fact_eval is None:
                 self._load_qafacteval(eval_cache_path)
             assert self.qa_fact_eval is not None
-            model_name = adapter_spec.model.replace("/", "_")
-            val = self.qa_fact_eval[model_name][(request_state.instance.id, pred)]
+            deployment = adapter_spec.model_deployment.replace("/", "_")
+            val = self.qa_fact_eval[deployment][(request_state.instance.id, pred)]
             result.append(Stat(MetricName("QAFactEval")).add(float(val)))
         except KeyError:
             pass

helm/benchmark/metrics/test_classification_metrics.py

@@ -7,7 +7,7 @@ from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.metrics.classification_metrics import ClassificationMetric
 from helm.benchmark.metrics.statistic import Stat
 from helm.benchmark.scenarios.scenario import Input, Instance, Output, Reference, CORRECT_TAG
-from helm.common.request import Request, RequestResult, Sequence
+from helm.common.request import Request, RequestResult, GeneratedOutput
 
 
 class _Option(NamedTuple):
@@ -26,9 +26,12 @@ def _request_state(prediction: str, options: List[_Option]):
         request_mode=None,
         train_trial_index=0,
         output_mapping=None,
-        request=Request(),
+        request=Request(model="openai/text-davinci-002", model_deployment="openai/text-davinci-002"),
         result=RequestResult(
-            success=True, embedding=[], completions=[Sequence(text=prediction, logprob=0.0, tokens=[])], cached=False
+            success=True,
+            embedding=[],
+            completions=[GeneratedOutput(text=prediction, logprob=0.0, tokens=[])],
+            cached=False,
         ),
         num_train_instances=0,
         prompt_truncated=False,
@@ -76,7 +79,7 @@ def test_evaluate_instances_binary_generation():
     ]
 
     assert_stats_equal(
-        metric.evaluate_instances(request_states),
+        metric.evaluate_instances(request_states, ""),
         _expected_stats(
             {
                 "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
@@ -106,7 +109,7 @@ def test_evaluate_instances_multi_class():
         _request_state("invalid", _options("c")),
     ]
     assert_stats_equal(
-        metric.evaluate_instances(request_states),
+        metric.evaluate_instances(request_states, ""),
         _expected_stats(
             {
                 "d": {"tp": 3, "fp": 1, "tn": 5, "fn": 1},
@@ -139,7 +142,7 @@ def test_evaluate_instances_multilabel():
     ]
 
     assert_stats_equal(
-        metric.evaluate_instances(request_states),
+        metric.evaluate_instances(request_states, ""),
         _expected_stats(
             {
                 "d": {"tp": 5, "fp": 1, "tn": 5, "fn": 0},

helm/benchmark/metrics/test_disinformation_metrics.py

@@ -0,0 +1,78 @@
+# Test metrics
+from typing import List
+
+import numpy as np
+import pytest
+from helm.benchmark.metrics.disinformation_metrics import _monte_carlo_entropy, _self_bleu
+from helm.common.request import GeneratedOutput, Token
+
+# Test tokens
+_TEST_1_TOKENS: List[Token] = [
+    Token("This", logprob=-0.25),
+    Token("is", logprob=-0.25),
+    Token("a", logprob=-0.25),
+    Token("test", logprob=-0.25),
+]
+_TEST_2_TOKENS: List[Token] = [
+    Token("This", logprob=-0.25),
+    Token("is", logprob=-0.25),
+    Token("another", logprob=-0.5),
+    Token("test", logprob=-0.25),
+]
+_TEST_EMPTY_TOKENS: List[Token] = []
+test_empty_str_tokens: List[Token] = [
+    Token("", logprob=0),
+]
+
+# Test Sequences (two standard, one with an empty token, and one with no tokens)
+_TEST_1 = GeneratedOutput(text="This is a test", logprob=-1, tokens=_TEST_1_TOKENS)
+_TEST_2 = GeneratedOutput(text="This is another test", logprob=-1.25, tokens=_TEST_2_TOKENS)
+_TEST_EMPTY = GeneratedOutput(text="", logprob=-float("nan"), tokens=_TEST_EMPTY_TOKENS)
+_TEST_EMPTY_STR = GeneratedOutput(text="", logprob=0, tokens=test_empty_str_tokens)
+
+
+# Test Self-BLEU
+def test_self_bleu_with_self():
+    score = _self_bleu([_TEST_1, _TEST_1])
+    assert score == pytest.approx(100)
+
+
+def test_self_blue_with_other():
+    score = _self_bleu([_TEST_1, _TEST_2])
+    assert 0 < score < 100
+
+
+def test_self_blue_one_sequence():
+    score = _self_bleu([_TEST_1])
+    assert score == 0
+
+
+def test_self_blue_one_full_one_empty():
+    score = _self_bleu([_TEST_1, _TEST_EMPTY_STR])
+    assert score == 0
+
+
+# Test MC Entropy
+def test_mc_entropy_with_self():
+    score = _monte_carlo_entropy([_TEST_1, _TEST_1])
+    assert score == pytest.approx(-_TEST_1.logprob)
+
+
+def test_mc_entropy_with_other():
+    score = _monte_carlo_entropy([_TEST_1, _TEST_2])
+    assert score == pytest.approx(-(_TEST_1.logprob + _TEST_2.logprob) / 2)
+
+
+def test_mc_entropy_one_sequence():
+    score = _monte_carlo_entropy([_TEST_1])
+    assert score == -_TEST_1.logprob
+
+
+def test_mc_entropy_one_full_one_empty():
+    score = _monte_carlo_entropy([_TEST_EMPTY_STR])
+    assert score == _TEST_EMPTY_STR.logprob
+
+
+def test_mc_entropy_with_no_tokens():
+    score = _monte_carlo_entropy([_TEST_EMPTY])
+    assert np.isnan(score)

helm/benchmark/metrics/test_evaluate_reference_metrics.py

@@ -0,0 +1,30 @@
+from helm.benchmark.metrics.evaluate_reference_metrics import (
+    exact_match,
+    exact_match_indicator,
+    final_number_exact_match,
+)
+
+
+def test_exact_match():
+    assert exact_match("33", "33") == 1
+    assert exact_match("33", "33 ") == 1
+    assert exact_match("33", "34") == 0
+
+
+def test_exact_match_indicator():
+    assert exact_match_indicator("33", "33") == 1
+    assert exact_match_indicator("33", "stuff 33") == 1
+    assert exact_match_indicator("stuff 33", "33") == 1
+    assert exact_match_indicator("33", "33 stuff") == 0
+
+
+def test_final_number_exact_match():
+    assert final_number_exact_match("33", "33") == 1
+    assert final_number_exact_match("33", "33 eggs.") == 1
+    assert final_number_exact_match("33", "-33") == 0
+    assert final_number_exact_match("-33", "-33") == 1
+    assert final_number_exact_match("The answer is 33", "\\boxed{33}") == 1
+    assert final_number_exact_match("The answer is 33", "\\boxed{33} and 34") == 0
+    assert final_number_exact_match("34.2", "2") == 0
+    assert final_number_exact_match("342", "342.") == 1
+    assert final_number_exact_match("3,420", "3420") == 1

helm/benchmark/metrics/test_metric.py

@@ -3,13 +3,13 @@ from .basic_metrics import get_num_bytes, convert_tokens_to_text
 
 
 def test_get_num_bytes():
-    tokens = [Token(text, 0, {}) for text in ["bytes:\\x99", "Hello", " world", "bytes:\\xe2\\x80"]]
+    tokens = [Token(text, 0) for text in ["bytes:\\x99", "Hello", " world", "bytes:\\xe2\\x80"]]
     assert get_num_bytes(tokens) == 14
 
 
 def test_convert_tokens_to_text():
     tokens = [
-        Token(text, 0, {})
+        Token(text, 0)
         for text in [
             "<|endoftext|>",
             "bytes:\\xe2\\x80",

helm/benchmark/metrics/tokens/auto_token_cost_estimator.py

@@ -39,5 +39,5 @@ class AutoTokenCostEstimator(TokenCostEstimator):
         """
         Estimate the number of tokens for a given request based on the organization.
         """
-        token_cost_estimator: TokenCostEstimator = self._get_estimator(request.model_organization)
+        token_cost_estimator: TokenCostEstimator = self._get_estimator(request.model_host)
         return token_cost_estimator.estimate_tokens(request, metric_service)

helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py

@@ -2,11 +2,19 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.common.request import Request
-from helm.proxy.token_counters.gooseai_token_counter import GooseAITokenCounter
 from .token_cost_estimator import TokenCostEstimator
 
 
 class GooseAITokenCostEstimator(TokenCostEstimator):
+    # From https://goose.ai/pricing: "the base price includes your first 25 tokens
+    # generated, and you can scale beyond that on a per-token basis."
+    BASE_PRICE_TOKENS: int = 25
+
+    @staticmethod
+    def account_for_base_tokens(num_tokens: int):
+        """Subtracts the number of tokens included in the base price."""
+        return max(num_tokens - GooseAITokenCostEstimator.BASE_PRICE_TOKENS, 0)
+
     def estimate_tokens(self, request: Request, metric_service: MetricService) -> int:
         """
         Estimate the number of generated tokens for a given request. Formula:
@@ -17,6 +25,8 @@ class GooseAITokenCostEstimator(TokenCostEstimator):
         """
         total_estimated_tokens: int = request.num_completions * request.max_tokens
         if request.echo_prompt:
-            window_service: WindowService = WindowServiceFactory.get_window_service(request.model, metric_service)
+            window_service: WindowService = WindowServiceFactory.get_window_service(
+                request.model_deployment, metric_service
+            )
             total_estimated_tokens += window_service.get_num_tokens(request.prompt)
-        return GooseAITokenCounter.account_for_base_tokens(total_estimated_tokens)
+        return GooseAITokenCostEstimator.account_for_base_tokens(total_estimated_tokens)

helm/benchmark/metrics/tokens/openai_token_cost_estimator.py

@@ -15,7 +15,7 @@ class OpenAITokenCostEstimator(TokenCostEstimator):
 
         Add num_tokens(prompt) if Request.echo_prompt is True.
         """
-        tokenizer: WindowService = WindowServiceFactory.get_window_service(request.model, metric_service)
+        tokenizer: WindowService = WindowServiceFactory.get_window_service(request.model_deployment, metric_service)
         num_prompt_tokens: int = tokenizer.get_num_tokens(request.prompt)
         total_estimated_tokens: int = num_prompt_tokens + request.num_completions * request.max_tokens
 

helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py

@@ -10,6 +10,8 @@ class TestAI21TokenCostEstimator:
 
     def test_estimate_tokens(self):
         request = Request(
+            model="openai/text-davinci-002",
+            model_deployment="openai/text-davinci-002",
             prompt="The Center for Research on Foundation Models (CRFM) is "
             "an interdisciplinary initiative born out of the Stanford "
             "Institute for Human-Centered Artificial Intelligence (HAI) "

helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py

@@ -12,7 +12,6 @@ from .openai_token_cost_estimator import OpenAITokenCostEstimator
 
 
 class TestOpenAITokenCostEstimator:
-
     # The following prompt has 51 tokens according to the GPT-2 tokenizer
     TEST_PROMPT: str = (
         "The Center for Research on Foundation Models (CRFM) is "
@@ -37,13 +36,21 @@ class TestOpenAITokenCostEstimator:
         self._mock_metric_service.tokenize = MagicMock(return_value=tokenization_request_result)
 
     def test_estimate_tokens(self):
-        request = Request(prompt=TestOpenAITokenCostEstimator.TEST_PROMPT, num_completions=3, max_tokens=100)
+        request = Request(
+            model="openai/text-davinci-002",
+            model_deployment="openai/text-davinci-002",
+            prompt=TestOpenAITokenCostEstimator.TEST_PROMPT,
+            num_completions=3,
+            max_tokens=100,
+        )
 
         # Prompt + max number of tokens from completions = 51 + 3 * 100
         assert self._token_cost_estimator.estimate_tokens(request, self._mock_metric_service) == 51 + 3 * 100
 
     def test_estimate_tokens_with_echo_prompt(self):
         request = Request(
+            model="openai/text-davinci-002",
+            model_deployment="openai/text-davinci-002",
             prompt=TestOpenAITokenCostEstimator.TEST_PROMPT,
             echo_prompt=True,
             num_completions=1,

helm/benchmark/metrics/toxicity_metrics.py

@@ -5,7 +5,7 @@ from helm.common.request import RequestResult
 from helm.common.hierarchical_logger import hlog
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.proxy.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
+from helm.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
 from .metric import Metric
 from .metric_name import MetricName
 from .metric_service import MetricService

helm/benchmark/metrics/toxicity_utils.py

@@ -0,0 +1,23 @@
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.common.perspective_api_request import PerspectiveAPIRequestResult, PerspectiveAPIRequest, ToxicityAttributes
+
+
+def compute_toxicity_score(text: str, metric_service: MetricService) -> float:
+    """
+    Compute the toxicity score of a given text using Perspective API.
+    Returns a value between 0 and 1 where a score 0.5 or greater is deemed toxic.
+    """
+    response: PerspectiveAPIRequestResult = metric_service.get_toxicity_scores(
+        request=PerspectiveAPIRequest(text_batch=[text])
+    )
+    attributes: ToxicityAttributes = response.text_to_toxicity_attributes[text]
+    assert attributes.toxicity_score is not None
+    return attributes.toxicity_score
+
+
+def is_prompt_toxic(text: str, metric_service: MetricService) -> bool:
+    """
+    Returns True, if the prompt is considered toxic, False otherwise.
+    """
+    score: float = compute_toxicity_score(text, metric_service)
+    return score >= 0.5