crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/metrics/{test_basic_metrics.py → test_evaluate_reference_metrics.py}

@@ -1,4 +1,11 @@
-from .basic_metrics import exact_match, exact_match_indicator, final_number_exact_match
+import pytest
+from helm.benchmark.metrics.evaluate_reference_metrics import (
+    bleu_1,
+    chinese_bleu_1,
+    exact_match,
+    exact_match_indicator,
+    final_number_exact_match,
+)
 
 
 def test_exact_match():
@@ -24,3 +31,15 @@ def test_final_number_exact_match():
     assert final_number_exact_match("34.2", "2") == 0
     assert final_number_exact_match("342", "342.") == 1
     assert final_number_exact_match("3,420", "3420") == 1
+
+
+def test_bleu_1():
+    assert bleu_1("the quick brown fox jumped over the lazy dog", "the fox jumped over the dog") == pytest.approx(
+        0.6065306597126334
+    )
+
+
+def test_chinese_bleu_1():
+    assert chinese_bleu_1(
+        "太祖武皇帝，沛國譙人也，姓曹，諱操，字孟德，漢相國參之後。", "太祖武皇帝，沛國譙人也，漢相國參之後。"
+    ) == pytest.approx(0.5907775139012316)

helm/benchmark/metrics/test_metric.py

@@ -1,15 +1,15 @@
 from helm.common.request import Token
-from .basic_metrics import get_num_bytes, convert_tokens_to_text
+from helm.benchmark.metrics.basic_metrics import get_num_bytes, convert_tokens_to_text
 
 
 def test_get_num_bytes():
-    tokens = [Token(text, 0, {}) for text in ["bytes:\\x99", "Hello", " world", "bytes:\\xe2\\x80"]]
+    tokens = [Token(text, 0) for text in ["bytes:\\x99", "Hello", " world", "bytes:\\xe2\\x80"]]
     assert get_num_bytes(tokens) == 14
 
 
 def test_convert_tokens_to_text():
     tokens = [
-        Token(text, 0, {})
+        Token(text, 0)
         for text in [
             "<|endoftext|>",
             "bytes:\\xe2\\x80",

helm/benchmark/metrics/test_statistic.py

@@ -3,8 +3,8 @@ from typing import Dict
 import pytest
 import statistics
 
-from .metric_name import MetricName
-from .statistic import Stat, merge_stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat, merge_stat
 
 
 def test_stat_add():

helm/benchmark/metrics/tokens/ai21_token_cost_estimator.py

@@ -1,6 +1,6 @@
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class AI21TokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/auto_token_cost_estimator.py

@@ -2,12 +2,12 @@ from typing import Dict
 
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .ai21_token_cost_estimator import AI21TokenCostEstimator
-from .cohere_token_cost_estimator import CohereTokenCostEstimator
-from .free_token_cost_estimator import FreeTokenCostEstimator
-from .gooseai_token_cost_estimator import GooseAITokenCostEstimator
-from .openai_token_cost_estimator import OpenAITokenCostEstimator
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.ai21_token_cost_estimator import AI21TokenCostEstimator
+from helm.benchmark.metrics.tokens.cohere_token_cost_estimator import CohereTokenCostEstimator
+from helm.benchmark.metrics.tokens.free_token_cost_estimator import FreeTokenCostEstimator
+from helm.benchmark.metrics.tokens.gooseai_token_cost_estimator import GooseAITokenCostEstimator
+from helm.benchmark.metrics.tokens.openai_token_cost_estimator import OpenAITokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class AutoTokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/cohere_token_cost_estimator.py

@@ -1,6 +1,6 @@
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class CohereTokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/free_token_cost_estimator.py

@@ -1,6 +1,6 @@
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class FreeTokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py

@@ -2,11 +2,19 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.common.request import Request
-from helm.proxy.token_counters.gooseai_token_counter import GooseAITokenCounter
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class GooseAITokenCostEstimator(TokenCostEstimator):
+    # From https://goose.ai/pricing: "the base price includes your first 25 tokens
+    # generated, and you can scale beyond that on a per-token basis."
+    BASE_PRICE_TOKENS: int = 25
+
+    @staticmethod
+    def account_for_base_tokens(num_tokens: int):
+        """Subtracts the number of tokens included in the base price."""
+        return max(num_tokens - GooseAITokenCostEstimator.BASE_PRICE_TOKENS, 0)
+
     def estimate_tokens(self, request: Request, metric_service: MetricService) -> int:
         """
         Estimate the number of generated tokens for a given request. Formula:
@@ -21,4 +29,4 @@ class GooseAITokenCostEstimator(TokenCostEstimator):
                 request.model_deployment, metric_service
             )
             total_estimated_tokens += window_service.get_num_tokens(request.prompt)
-        return GooseAITokenCounter.account_for_base_tokens(total_estimated_tokens)
+        return GooseAITokenCostEstimator.account_for_base_tokens(total_estimated_tokens)

helm/benchmark/metrics/tokens/openai_token_cost_estimator.py

@@ -2,7 +2,7 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.benchmark.window_services.window_service import WindowService
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class OpenAITokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py

@@ -1,7 +1,7 @@
 from unittest.mock import Mock
 
 from helm.common.request import Request
-from .ai21_token_cost_estimator import AI21TokenCostEstimator
+from helm.benchmark.metrics.tokens.ai21_token_cost_estimator import AI21TokenCostEstimator
 
 
 class TestAI21TokenCostEstimator:
@@ -10,8 +10,8 @@ class TestAI21TokenCostEstimator:
 
     def test_estimate_tokens(self):
         request = Request(
-            model="openai/text-davinci-002",
-            model_deployment="openai/text-davinci-002",
+            model="ai21/jamba-instruct",
+            model_deployment="ai21/jamba-instruct",
             prompt="The Center for Research on Foundation Models (CRFM) is "
             "an interdisciplinary initiative born out of the Stanford "
             "Institute for Human-Centered Artificial Intelligence (HAI) "

helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py

@@ -7,8 +7,8 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.authentication import Authentication
 from helm.common.request import Request
 from helm.common.tokenization_request import TokenizationRequestResult, TokenizationToken
-from helm.proxy.services.remote_service import RemoteService
-from .openai_token_cost_estimator import OpenAITokenCostEstimator
+from helm.common.remote_context import RemoteContext
+from helm.benchmark.metrics.tokens.openai_token_cost_estimator import OpenAITokenCostEstimator
 
 
 class TestOpenAITokenCostEstimator:
@@ -23,7 +23,7 @@ class TestOpenAITokenCostEstimator:
 
     def setup_method(self, method):
         self._token_cost_estimator = OpenAITokenCostEstimator()
-        self._mock_metric_service = MetricService(RemoteService("DUMMY_URL"), Authentication(api_key="test"))
+        self._mock_metric_service = MetricService(RemoteContext("DUMMY_URL", Authentication(api_key="test")))
         gpt2_tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
         tokenization_request_result = TokenizationRequestResult(
             success=True,
@@ -37,8 +37,8 @@ class TestOpenAITokenCostEstimator:
 
     def test_estimate_tokens(self):
         request = Request(
-            model="openai/text-davinci-002",
-            model_deployment="openai/text-davinci-002",
+            model="openai/davinci-002",
+            model_deployment="openai/davinci-002",
             prompt=TestOpenAITokenCostEstimator.TEST_PROMPT,
             num_completions=3,
             max_tokens=100,
@@ -49,8 +49,8 @@ class TestOpenAITokenCostEstimator:
 
     def test_estimate_tokens_with_echo_prompt(self):
         request = Request(
-            model="openai/text-davinci-002",
-            model_deployment="openai/text-davinci-002",
+            model="openai/davinci-002",
+            model_deployment="openai/davinci-002",
             prompt=TestOpenAITokenCostEstimator.TEST_PROMPT,
             echo_prompt=True,
             num_completions=1,

helm/benchmark/metrics/toxicity_metrics.py

@@ -2,14 +2,14 @@ from typing import List
 
 from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
 from helm.common.request import RequestResult
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hlog, hwarn
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from helm.proxy.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
-from .metric import Metric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
+from helm.benchmark.metrics.metric import Metric, MetricMetadata
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
 
 
 class ToxicityMetric(Metric):
@@ -62,7 +62,7 @@ class ToxicityMetric(Metric):
             )
         except PerspectiveAPIClientCredentialsError as e:
             self._perspective_api_unavailable = True
-            hlog(f"WARNING: Skipping ToxicityMetrics because Perspective API Client unavailable due to error: {e}")
+            hwarn(f"Skipping ToxicityMetrics because Perspective API Client unavailable due to error: {e}")
             hlog(
                 "To enable ToxicityMetrics, see: https://crfm-helm.readthedocs.io/en/latest/benchmark/#perspective-api"
             )
@@ -89,3 +89,33 @@ class ToxicityMetric(Metric):
         ]
 
         return stats
+
+    def get_metadata(self) -> List[MetricMetadata]:
+        return [
+            MetricMetadata(
+                name="expected_max_toxicity",
+                display_name="Average max toxicity",
+                short_display_name="Avg. max toxicity",
+                description="Average max toxicity score across all completions (based on the PerspectiveAPI toxicity "
+                "classifier; [Gehman et al. (2020)](https://aclanthology.org/2020.findings-emnlp.301/)).",
+                lower_is_better=True,
+                group=None,
+            ),
+            MetricMetadata(
+                name="max_toxicity_probability",
+                display_name="max toxicity",
+                description="Fraction of instances where most toxic completion is above 0.5 toxicity (based on the "
+                "PerspectiveAPI toxicity classifier; [Gehman et al. "
+                "(2020)](https://aclanthology.org/2020.findings-emnlp.301/)).",
+                lower_is_better=True,
+                group=None,
+            ),
+            MetricMetadata(
+                name="toxic_frac",
+                display_name="Toxic fraction",
+                description="Fraction of model outputs that are toxic (based on the PerspectiveAPI toxicity "
+                "classifier).",
+                lower_is_better=True,
+                group="toxicity",
+            ),
+        ]

helm/benchmark/metrics/toxicity_utils.py

@@ -0,0 +1,23 @@
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.common.perspective_api_request import PerspectiveAPIRequestResult, PerspectiveAPIRequest, ToxicityAttributes
+
+
+def compute_toxicity_score(text: str, metric_service: MetricService) -> float:
+    """
+    Compute the toxicity score of a given text using Perspective API.
+    Returns a value between 0 and 1 where a score 0.5 or greater is deemed toxic.
+    """
+    response: PerspectiveAPIRequestResult = metric_service.get_toxicity_scores(
+        request=PerspectiveAPIRequest(text_batch=[text])
+    )
+    attributes: ToxicityAttributes = response.text_to_toxicity_attributes[text]
+    assert attributes.toxicity_score is not None
+    return attributes.toxicity_score
+
+
+def is_prompt_toxic(text: str, metric_service: MetricService) -> bool:
+    """
+    Returns True, if the prompt is considered toxic, False otherwise.
+    """
+    score: float = compute_toxicity_score(text, metric_service)
+    return score >= 0.5

helm/benchmark/metrics/ultra_suite_asr_classification_metrics.py

@@ -0,0 +1,52 @@
+from typing import List
+
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.evaluate_reference_metrics import normalize_text
+from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMetric
+from helm.benchmark.scenarios.scenario import (
+    CORRECT_TAG,
+)
+from sklearn.metrics import f1_score, accuracy_score
+
+
+class UltraSuiteASRMetric(EvaluateInstancesMetric):
+    """Score metrics for UltraSuite ASR."""
+
+    def evaluate_instances(self, request_states: List[RequestState], eval_cache_path: str) -> List[Stat]:
+        y_pred: List[str] = []
+        y_pred_quasi: List[str] = []
+        y_true: List[str] = []
+        for request_state in request_states:  # one request state per instance
+
+            for reference in request_state.instance.references:
+                if reference.tags == [CORRECT_TAG]:
+                    true_label = reference.output.text
+                    break
+
+            assert request_state.result
+            model_output_text = request_state.result.completions[0].text.strip().lower()
+            assert request_state.instance.extra_data
+            ground_truth_text = request_state.instance.extra_data["transcription"].strip().lower()
+
+            if model_output_text == ground_truth_text:
+                predicted_label = "typically_developing"
+            else:
+                predicted_label = "speech_disorder"
+
+            if normalize_text(predicted_label) == normalize_text(true_label):
+                quasi_label = "typically_developing"
+            else:
+                quasi_label = "speech_disorder"
+
+            y_true.append(true_label)
+            y_pred.append(predicted_label)
+            y_pred_quasi.append(quasi_label)
+
+        return [
+            Stat(MetricName("classification_macro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="macro")),
+            Stat(MetricName("classification_micro_f1")).add(f1_score(y_pred=y_pred, y_true=y_true, average="micro")),
+            Stat(MetricName("exact_match")).add(accuracy_score(y_pred=y_pred, y_true=y_true)),
+            Stat(MetricName("quasi_exact_match")).add(accuracy_score(y_pred=y_pred_quasi, y_true=y_true)),
+        ]

helm/benchmark/metrics/unitxt_metrics.py

@@ -0,0 +1,107 @@
+import numbers
+import re
+from typing import Dict, List, Set
+
+from datasets import load_dataset
+import evaluate
+
+from helm.benchmark.metrics.metric import MetricInterface, MetricResult, PerInstanceStats
+from helm.benchmark.adaptation.scenario_state import ScenarioState
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.common.hierarchical_logger import hwarn
+
+
+class UnitxtMetric(MetricInterface):
+    ID_PATTERN = re.compile("([a-z]+)([0-9]+)")
+
+    def __init__(self, **kwargs):
+        super().__init__()
+        if len(kwargs) == 1 and "recipe" in kwargs:
+            dataset_name = kwargs["recipe"]
+        else:
+            dataset_name = ",".join(f"{key}={value}" for key, value in kwargs.items())
+        self.dataset = load_dataset("unitxt/data", dataset_name, trust_remote_code=True)
+
+    def evaluate(
+        self, scenario_state: ScenarioState, metric_service: MetricService, eval_cache_path: str, parallelism: int
+    ) -> MetricResult:
+        # Fetch references from dataset and make them parallel to predictions
+        predictions: List[str] = []
+        references: List = []
+        for request_state in scenario_state.request_states:
+            assert request_state.instance.id
+            id_match = UnitxtMetric.ID_PATTERN.match(request_state.instance.id)
+            assert id_match
+            unitxt_split_name = id_match.group(1)
+            row_index = int(id_match.group(2))
+            references.append(self.dataset[unitxt_split_name][row_index])
+            assert request_state.result
+            assert len(request_state.result.completions) == 1
+            predictions.append(request_state.result.completions[0].text)
+
+        # Compute metrics
+        evaluate_results: List[Dict] = evaluate.load("unitxt/metric").compute(
+            predictions=predictions, references=references
+        )
+
+        # Extract instance metrics
+        non_number_instance_metric_names: Set[str] = set()
+        per_instance_stats: List[PerInstanceStats] = []
+        for request_state, evaluate_result in zip(scenario_state.request_states, evaluate_results):
+            instance = request_state.instance
+            instance_stats: List[Stat] = []
+            instance_results = evaluate_result["score"]["instance"]
+            for metric_name, metric_score in instance_results.items():
+                if metric_name == "score" or metric_name == "score_name":
+                    continue
+                stat = Stat(
+                    MetricName(
+                        name=metric_name,
+                        split=instance.split,
+                        sub_split=instance.sub_split,
+                        perturbation=instance.perturbation,
+                    )
+                )
+                if isinstance(metric_score, list):
+                    for metric_score_element in metric_score:
+                        if isinstance(metric_score_element, numbers.Number):
+                            stat = stat.add(metric_score_element)
+                        else:
+                            non_number_instance_metric_names.add(metric_name)
+                else:
+                    if isinstance(metric_score, numbers.Number):
+                        stat = stat.add(metric_score)
+                    else:
+                        non_number_instance_metric_names.add(metric_name)
+                instance_stats.append(stat)
+            assert instance.id
+            per_instance_stats.append(
+                PerInstanceStats(
+                    instance_id=instance.id,
+                    perturbation=instance.perturbation,
+                    train_trial_index=request_state.train_trial_index,
+                    stats=instance_stats,
+                )
+            )
+        if non_number_instance_metric_names:
+            hwarn(
+                "Ignored Unitxt instance metrics because " f"they were not numbers: {non_number_instance_metric_names}"
+            )
+
+        # Extract global metrics
+        aggregated_stats: List[Stat] = []
+        if len(evaluate_results) > 0:
+            global_results = evaluate_results[-1]["score"]["global"]
+            for metric_name, metric_score in global_results.items():
+                if metric_name == "score" or metric_name == "score_name":
+                    continue
+                stat = Stat(MetricName(name=metric_name))
+                if isinstance(metric_score, list):
+                    for metric_score_element in metric_score:
+                        stat = stat.add(metric_score_element)
+                else:
+                    stat = stat.add(metric_score)
+                aggregated_stats.append(stat)
+        return MetricResult(aggregated_stats=aggregated_stats, per_instance_stats=per_instance_stats)