crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/metrics/ruler_qa_metrics.py

@@ -0,0 +1,34 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.scenarios.scenario import CORRECT_TAG
+
+
+class RulerQAMetric(Metric):
+    """Accuracy metric for Ruler QA Scenarios.
+
+    Adapted from: https://github.com/NVIDIA/RULER/blob/1c45e5c60273e0ae9e3099137bf0eec6f0395f84/scripts/eval/synthetic/constants.py#L25
+    """  # noqa: E501
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.result
+        assert len(request_state.result.completions) == 1
+        response_text = request_state.result.completions[0].text
+        assert all(
+            len(reference.tags) == 1 and reference.tags[0] == CORRECT_TAG
+            for reference in request_state.instance.references
+        )
+        reference_texts = [reference.output.text for reference in request_state.instance.references]
+        score = max([1.0 if r.lower() in response_text.lower() else 0.0 for r in reference_texts])
+        return [Stat(MetricName("ruler_string_match_part")).add(score)]

helm/benchmark/metrics/{bhasa_metrics.py → seahelm_metrics.py}

@@ -14,7 +14,7 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.metrics.statistic import Stat
 
 
-class BhasaMachineTranslationMetric(Metric):
+class SEAHELMMachineTranslationMetric(Metric):
     """Machine Translation Metrics
 
     This class computes the following standard machine translation metrics
@@ -74,8 +74,8 @@ class BhasaMachineTranslationMetric(Metric):
         return result
 
 
-class BhasaQAMetric(Metric):
-    """Bhasa QA Metrics
+class SEAHELMQAMetric(Metric):
+    """SEAHELM QA Metrics
 
     This class computes the following standard SQuAD v1.1 metrics
 

helm/benchmark/metrics/seahelm_metrics_specs.py

@@ -0,0 +1,10 @@
+from typing import Any, Dict, List
+from helm.benchmark.metrics.metric import MetricSpec
+
+
+def get_seahelm_machine_translation_metric_specs() -> List[MetricSpec]:
+    return [MetricSpec(class_name="helm.benchmark.metrics.seahelm_metrics.SEAHELMMachineTranslationMetric")]
+
+
+def get_seahelm_qa_metric_specs(args: Dict[str, Any]) -> List[MetricSpec]:
+    return [MetricSpec(class_name="helm.benchmark.metrics.seahelm_metrics.SEAHELMQAMetric", args=args)]

helm/benchmark/metrics/spider_metrics.py

@@ -0,0 +1,7 @@
+from helm.benchmark.metrics.bird_sql_metrics import BirdSQLMetric
+
+
+class SpiderMetric(BirdSQLMetric):
+    """Score metrics for Spider. Based on Bird-SQL."""
+
+    ANNOTATOR_NAME = "spider"

helm/benchmark/metrics/starr_patient_instructions_metrics.py

@@ -0,0 +1,14 @@
+from helm.benchmark.annotation.starr_patient_instructions_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
+
+
+class StarrPatientInstructionsMetric(LLMJuryMetric):
+    """Score metrics for StarrPatientInstructions."""
+
+    def __init__(self):
+        super().__init__(
+            metric_name="starr_patient_instructions_accuracy",
+            scenario_name="starr_patient_instructions",
+            annotator_models=ANNOTATOR_MODELS,
+            default_score=1.0,
+        )

helm/benchmark/metrics/statistic.py

@@ -2,7 +2,7 @@ from dataclasses import dataclass, replace
 import math
 from typing import Dict, Optional
 
-from .metric_name import MetricName
+from helm.benchmark.metrics.metric_name import MetricName
 
 
 @dataclass

helm/benchmark/metrics/summac/model_summac.py

@@ -10,7 +10,7 @@ import numpy.typing as npt
 import torch
 import os
 import json
-from . import utils_misc
+from helm.benchmark.metrics.summac import utils_misc
 
 model_map = {
     "snli-base": {"model_card": "boychaboy/SNLI_roberta-base", "entailment_idx": 0, "contradiction_idx": 2},
@@ -169,10 +169,9 @@ class SummaCImager:
                 batch_tokens = self.tokenizer.batch_encode_plus(
                     list(zip(batch_prems, batch_hypos)),
                     padding=True,
-                    truncation=True,
+                    truncation="only_first",
                     max_length=self.max_input_length,
                     return_tensors="pt",
-                    truncation_strategy="only_first",
                 )
                 batch_tokens = {k: v.to(self.device) for k, v in batch_tokens.items()}
                 with torch.no_grad():

helm/benchmark/metrics/summarization_critique_metrics.py

@@ -2,10 +2,10 @@ from typing import Dict, List
 
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
-from .metric import Metric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
 from helm.common.critique_request import CritiqueTaskTemplate, CritiqueQuestionTemplate, CritiqueRequest, QuestionType
 
 

helm/benchmark/metrics/summarization_metrics.py

@@ -16,12 +16,16 @@ from helm.benchmark.metrics.evaluate_reference_metrics import get_rouge_function
 from helm.common.hierarchical_logger import hlog
 from helm.common.general import ensure_file_downloaded
 from helm.common.optional_dependencies import handle_module_not_found_error
-from .metric import Metric, MetricResult
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
-from .summac.model_summac import SummaCZS
-from bert_score import BERTScorer
+from helm.benchmark.metrics.metric import Metric, MetricResult
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.summac.model_summac import SummaCZS
+
+try:
+    from bert_score import BERTScorer  # type: ignore
+except ModuleNotFoundError as e:
+    handle_module_not_found_error(e, ["summarization"])
 
 
 QAFACTEVAL_URL: str = (
@@ -43,7 +47,14 @@ class SummarizationMetric(Metric):
     4. Faithfulness (SummaC)
     """
 
-    def __init__(self, task: str, device: str = "cpu"):
+    def __init__(
+        self,
+        task: str,
+        language: str = "en",
+        device: str = "cpu",
+        bertscore_model: str = "microsoft/deberta-large-mnli",
+        rescale_with_baseline: bool = True,
+    ):
         self.rouge_fns = {
             "rouge_1": get_rouge_function("rouge1"),
             "rouge_2": get_rouge_function("rouge2"),
@@ -56,7 +67,7 @@ class SummarizationMetric(Metric):
             spacy.cli.download("en_core_web_sm")
 
         try:
-            from summ_eval.data_stats_metric import DataStatsMetric
+            from summ_eval.data_stats_metric import DataStatsMetric  # type: ignore
         except ModuleNotFoundError as e:
             handle_module_not_found_error(e, ["summarization"])
 
@@ -71,7 +82,7 @@ class SummarizationMetric(Metric):
         else:
             self.compute_bertscore = True
             self.bert_scorer = BERTScorer(
-                model_type="microsoft/deberta-large-mnli", lang="en", rescale_with_baseline=True, device=device
+                model_type=bertscore_model, lang=language, rescale_with_baseline=rescale_with_baseline, device=device
             )
             # Need GPU for faithfulness metrics since they are model-based.
             self.compute_faithfulness = True

helm/benchmark/metrics/test_bias_metrics.py

@@ -1,7 +1,11 @@
 from dataclasses import dataclass
 from typing import Callable, List, Optional
 
-from .bias_metrics import BiasMetric
+from helm.benchmark.metrics.bias_metrics import BiasMetric
+from helm.benchmark.metrics.nltk_helper import install_nltk_resources
+
+
+install_nltk_resources()
 
 
 @dataclass(frozen=True)

helm/benchmark/metrics/test_classification_metrics.py

@@ -38,12 +38,14 @@ def _request_state(prediction: str, options: List[_Option]):
     )
 
 
-def assert_stats_equal(actual_stats: List[Stat], expected_values: Dict[str, float]):
-    actual_values = {stat.name.name: stat.mean for stat in actual_stats}
-    assert actual_values == approx(expected_values)
+def get_stat_value(stats: List[Stat], stat_name: str):
+    for stat in stats:
+        if stat.name.name == stat_name:
+            return stat.mean
+    raise ValueError(f"No stat with name {stat_name}")
 
 
-def _expected_stats(all_classes_counts: Dict[str, Dict[str, int]]):
+def compute_stats(all_classes_counts: Dict[str, Dict[str, int]]):
     micro_counts: Dict[str, int] = defaultdict(int)
     for class_counts in all_classes_counts.values():
         for key, class_count in class_counts.items():
@@ -52,102 +54,172 @@ def _expected_stats(all_classes_counts: Dict[str, Dict[str, int]]):
     micro_recall = micro_counts["tp"] / (micro_counts["tp"] + micro_counts["fn"])
     micro_f1 = 2 * (micro_precision * micro_recall) / (micro_precision + micro_recall)
 
-    class_f1: List[float] = []
-    for class_counts in all_classes_counts.values():
+    class_f1: Dict[str, float] = {}
+    for class_name, class_counts in all_classes_counts.items():
         class_precision = class_counts["tp"] / (class_counts["tp"] + class_counts["fp"])
         class_recall = class_counts["tp"] / (class_counts["tp"] + class_counts["fn"])
-        class_f1.append(2 * (class_precision * class_recall) / (class_precision + class_recall))
-    macro_f1 = sum(class_f1) / len(class_f1)
-
-    return {
-        "classification_micro_f1": micro_f1,
-        "classification_macro_f1": macro_f1,
+        class_f1[class_name] = 2 * (class_precision * class_recall) / (class_precision + class_recall)
+    macro_f1 = sum(class_f1.values()) / len(class_f1)
+    class_name_to_support = {
+        class_name: class_counts["tp"] + class_counts["fn"] for class_name, class_counts in all_classes_counts.items()
     }
+    weighted_f1 = sum(class_f1[class_name] * support for class_name, support in class_name_to_support.items()) / sum(
+        support for support in class_name_to_support.values()
+    )
 
+    stats = {
+        "macro_f1": macro_f1,
+        "micro_f1": micro_f1,
+        "weighted_f1": weighted_f1,
+    }
+    for class_name, class_f1_score in class_f1.items():
+        stats[f"{class_name}_f1"] = class_f1_score
+    return stats
 
-def test_evaluate_instances_binary_generation():
-    metric = ClassificationMetric(delimiter=None)
 
+def test_evaluate_instances_default_parameters():
     request_states = [
         _request_state("yes", [_Option("yes", True)]),
+        _request_state("yes ", [_Option("yes", True)]),
+        _request_state("yeS", [_Option("yes", True)]),
+        _request_state("yes", [_Option("no", True)]),
+        _request_state("no", [_Option("yes", True)]),
+        _request_state("no", [_Option("no", True)]),
+        _request_state("invalid", [_Option("no", True)]),
+    ]
+
+    expected_stats = compute_stats(
+        {
+            "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
+            "no": {"tp": 1, "fp": 1, "tn": 3, "fn": 2},
+        }
+    )
+
+    actual_stats = ClassificationMetric().evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+
+
+def test_evaluate_instances_yes_and_no():
+    labels = ["yes", "no"]
+    request_states = [
         _request_state("yes", [_Option("yes", True)]),
-        _request_state("yes", [_Option("yes", True)]),
+        _request_state("yes ", [_Option("yes", True)]),
+        _request_state("yeS", [_Option("yes", True)]),
         _request_state("yes", [_Option("no", True)]),
         _request_state("no", [_Option("yes", True)]),
         _request_state("no", [_Option("no", True)]),
         _request_state("invalid", [_Option("no", True)]),
     ]
 
-    assert_stats_equal(
-        metric.evaluate_instances(request_states, ""),
-        _expected_stats(
-            {
-                "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
-                "no": {"tp": 1, "fp": 1, "tn": 3, "fn": 2},
-            }
-        ),
+    expected_stats = compute_stats(
+        {
+            "yes": {"tp": 3, "fp": 1, "tn": 2, "fn": 1},
+            "no": {"tp": 1, "fp": 1, "tn": 3, "fn": 2},
+        }
     )
 
+    actual_stats = ClassificationMetric(
+        scores=["f1"], averages=["macro", "micro", "weighted", None], labels=labels
+    ).evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+    actual_weighted_f1 = get_stat_value(actual_stats, "classification_weighted_f1")
+    assert actual_weighted_f1 == approx(expected_stats["weighted_f1"])
+    actual_yes_f1 = get_stat_value(actual_stats, "classification_yes_f1")
+    assert actual_yes_f1 == approx(expected_stats["yes_f1"])
+    actual_no_f1 = get_stat_value(actual_stats, "classification_no_f1")
+    assert actual_no_f1 == approx(expected_stats["no_f1"])
+
 
 def test_evaluate_instances_multi_class():
-    # Note: no "a" because it would get filtered out by normalize_text()
-    metric = ClassificationMetric(delimiter=None)
+    labels = ["a", "b", "c"]
 
-    def _options(correct: str):
-        return [_Option(text, text == correct) for text in ["d", "b", "c"]]
+    def _gold_label(correct: str):
+        return [_Option(text, text == correct) for text in labels]
 
     request_states = [
-        _request_state("d", _options("d")),
-        _request_state("d", _options("d")),
-        _request_state("d", _options("d")),
-        _request_state("d", _options("b")),
-        _request_state("b", _options("b")),
-        _request_state("b", _options("b")),
-        _request_state("b", _options("c")),
-        _request_state("c", _options("d")),
-        _request_state("c", _options("c")),
-        _request_state("invalid", _options("c")),
+        _request_state("a", _gold_label("a")),
+        _request_state("a", _gold_label("a")),
+        _request_state("a", _gold_label("a")),
+        _request_state("a", _gold_label("b")),
+        _request_state("b", _gold_label("b")),
+        _request_state("b", _gold_label("b")),
+        _request_state("b", _gold_label("c")),
+        _request_state("c", _gold_label("a")),
+        _request_state("c", _gold_label("c")),
+        _request_state("invalid", _gold_label("c")),
     ]
-    assert_stats_equal(
-        metric.evaluate_instances(request_states, ""),
-        _expected_stats(
-            {
-                "d": {"tp": 3, "fp": 1, "tn": 5, "fn": 1},
-                "b": {"tp": 2, "fp": 1, "tn": 6, "fn": 1},
-                "c": {"tp": 1, "fp": 1, "tn": 6, "fn": 2},
-            }
-        ),
+
+    expected_stats = compute_stats(
+        {
+            "a": {"tp": 3, "fp": 1, "tn": 5, "fn": 1},
+            "b": {"tp": 2, "fp": 1, "tn": 6, "fn": 1},
+            "c": {"tp": 1, "fp": 1, "tn": 6, "fn": 2},
+        }
     )
 
+    actual_stats = ClassificationMetric(
+        scores=["f1"], averages=["macro", "micro", "weighted", None], labels=labels
+    ).evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+    actual_weighted_f1 = get_stat_value(actual_stats, "classification_weighted_f1")
+    assert actual_weighted_f1 == approx(expected_stats["weighted_f1"])
+    actual_a_f1 = get_stat_value(actual_stats, "classification_a_f1")
+    assert actual_a_f1 == approx(expected_stats["a_f1"])
+    actual_b_f1 = get_stat_value(actual_stats, "classification_b_f1")
+    assert actual_b_f1 == approx(expected_stats["b_f1"])
+    actual_c_f1 = get_stat_value(actual_stats, "classification_c_f1")
+    assert actual_c_f1 == approx(expected_stats["c_f1"])
+
 
 def test_evaluate_instances_multilabel():
-    # Note: no "a" because it would get filtered out by normalize_text()
-    metric = ClassificationMetric(delimiter=",")
+    labels = ["a", "b", "c"]
 
-    def _options(correct: List[str]):
-        return [_Option(text, text in correct) for text in ["d", "b", "c"]]
+    def _gold_labels(correct: List[str]):
+        return [_Option(text, text in correct) for text in labels]
 
     request_states = [
-        _request_state("d,b", _options(["d", "b"])),
-        _request_state("d,b", _options(["d", "c"])),
-        _request_state("d", _options(["d"])),
-        _request_state("c", _options(["b"])),
-        _request_state("b", _options(["b", "c"])),
-        _request_state("d,b", _options(["c"])),
-        _request_state("d,c", _options(["d"])),
-        _request_state("d,b,c", _options(["d", "b", "c"])),
+        _request_state("a,b", _gold_labels(["a", "b"])),
+        _request_state("a,b", _gold_labels(["a", "c"])),
+        _request_state("a", _gold_labels(["a"])),
+        _request_state("c", _gold_labels(["b"])),
+        _request_state("b", _gold_labels(["b", "c"])),
+        _request_state("a,b", _gold_labels(["c"])),
+        _request_state("a,c", _gold_labels(["a"])),
+        _request_state("a,b,c", _gold_labels(["a", "b", "c"])),
         _request_state("", []),
         _request_state("n/a", []),
-        _request_state("invalid", _options(["c"])),
+        _request_state("invalid", _gold_labels(["c"])),
     ]
 
-    assert_stats_equal(
-        metric.evaluate_instances(request_states, ""),
-        _expected_stats(
-            {
-                "d": {"tp": 5, "fp": 1, "tn": 5, "fn": 0},
-                "b": {"tp": 3, "fp": 2, "tn": 5, "fn": 1},
-                "c": {"tp": 1, "fp": 2, "tn": 4, "fn": 4},
-            }
-        ),
+    expected_stats = compute_stats(
+        {
+            "a": {"tp": 5, "fp": 1, "tn": 5, "fn": 0},
+            "b": {"tp": 3, "fp": 2, "tn": 5, "fn": 1},
+            "c": {"tp": 1, "fp": 2, "tn": 4, "fn": 4},
+        }
     )
+
+    actual_stats = ClassificationMetric(
+        scores=["f1"], averages=["macro", "micro", "weighted", None], labels=labels, delimiter=","
+    ).evaluate_instances(request_states, "")
+    actual_macro_f1 = get_stat_value(actual_stats, "classification_macro_f1")
+    assert actual_macro_f1 == approx(expected_stats["macro_f1"])
+    actual_micro_f1 = get_stat_value(actual_stats, "classification_micro_f1")
+    assert actual_micro_f1 == approx(expected_stats["micro_f1"])
+    actual_weighted_f1 = get_stat_value(actual_stats, "classification_weighted_f1")
+    assert actual_weighted_f1 == approx(expected_stats["weighted_f1"])
+    actual_a_f1 = get_stat_value(actual_stats, "classification_a_f1")
+    assert actual_a_f1 == approx(expected_stats["a_f1"])
+    actual_b_f1 = get_stat_value(actual_stats, "classification_b_f1")
+    assert actual_b_f1 == approx(expected_stats["b_f1"])
+    actual_c_f1 = get_stat_value(actual_stats, "classification_c_f1")
+    assert actual_c_f1 == approx(expected_stats["c_f1"])

helm/benchmark/metrics/test_evaluate_reference_metrics.py

@@ -1,4 +1,7 @@
+import pytest
 from helm.benchmark.metrics.evaluate_reference_metrics import (
+    bleu_1,
+    chinese_bleu_1,
     exact_match,
     exact_match_indicator,
     final_number_exact_match,
@@ -28,3 +31,15 @@ def test_final_number_exact_match():
     assert final_number_exact_match("34.2", "2") == 0
     assert final_number_exact_match("342", "342.") == 1
     assert final_number_exact_match("3,420", "3420") == 1
+
+
+def test_bleu_1():
+    assert bleu_1("the quick brown fox jumped over the lazy dog", "the fox jumped over the dog") == pytest.approx(
+        0.6065306597126334
+    )
+
+
+def test_chinese_bleu_1():
+    assert chinese_bleu_1(
+        "太祖武皇帝，沛國譙人也，姓曹，諱操，字孟德，漢相國參之後。", "太祖武皇帝，沛國譙人也，漢相國參之後。"
+    ) == pytest.approx(0.5907775139012316)

helm/benchmark/metrics/test_metric.py

@@ -1,5 +1,5 @@
 from helm.common.request import Token
-from .basic_metrics import get_num_bytes, convert_tokens_to_text
+from helm.benchmark.metrics.basic_metrics import get_num_bytes, convert_tokens_to_text
 
 
 def test_get_num_bytes():

helm/benchmark/metrics/test_statistic.py

@@ -3,8 +3,8 @@ from typing import Dict
 import pytest
 import statistics
 
-from .metric_name import MetricName
-from .statistic import Stat, merge_stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat, merge_stat
 
 
 def test_stat_add():

helm/benchmark/metrics/tokens/ai21_token_cost_estimator.py

@@ -1,6 +1,6 @@
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class AI21TokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/auto_token_cost_estimator.py

@@ -2,12 +2,12 @@ from typing import Dict
 
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .ai21_token_cost_estimator import AI21TokenCostEstimator
-from .cohere_token_cost_estimator import CohereTokenCostEstimator
-from .free_token_cost_estimator import FreeTokenCostEstimator
-from .gooseai_token_cost_estimator import GooseAITokenCostEstimator
-from .openai_token_cost_estimator import OpenAITokenCostEstimator
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.ai21_token_cost_estimator import AI21TokenCostEstimator
+from helm.benchmark.metrics.tokens.cohere_token_cost_estimator import CohereTokenCostEstimator
+from helm.benchmark.metrics.tokens.free_token_cost_estimator import FreeTokenCostEstimator
+from helm.benchmark.metrics.tokens.gooseai_token_cost_estimator import GooseAITokenCostEstimator
+from helm.benchmark.metrics.tokens.openai_token_cost_estimator import OpenAITokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class AutoTokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/cohere_token_cost_estimator.py

@@ -1,6 +1,6 @@
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class CohereTokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/free_token_cost_estimator.py

@@ -1,6 +1,6 @@
 from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class FreeTokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/gooseai_token_cost_estimator.py

@@ -2,7 +2,7 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.common.request import Request
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class GooseAITokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/openai_token_cost_estimator.py

@@ -2,7 +2,7 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.request import Request
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.benchmark.window_services.window_service import WindowService
-from .token_cost_estimator import TokenCostEstimator
+from helm.benchmark.metrics.tokens.token_cost_estimator import TokenCostEstimator
 
 
 class OpenAITokenCostEstimator(TokenCostEstimator):

helm/benchmark/metrics/tokens/test_ai21_token_cost_estimator.py

@@ -1,7 +1,7 @@
 from unittest.mock import Mock
 
 from helm.common.request import Request
-from .ai21_token_cost_estimator import AI21TokenCostEstimator
+from helm.benchmark.metrics.tokens.ai21_token_cost_estimator import AI21TokenCostEstimator
 
 
 class TestAI21TokenCostEstimator:

helm/benchmark/metrics/tokens/test_openai_token_cost_estimator.py

@@ -7,8 +7,8 @@ from helm.benchmark.metrics.metric_service import MetricService
 from helm.common.authentication import Authentication
 from helm.common.request import Request
 from helm.common.tokenization_request import TokenizationRequestResult, TokenizationToken
-from helm.proxy.services.remote_service import RemoteService
-from .openai_token_cost_estimator import OpenAITokenCostEstimator
+from helm.common.remote_context import RemoteContext
+from helm.benchmark.metrics.tokens.openai_token_cost_estimator import OpenAITokenCostEstimator
 
 
 class TestOpenAITokenCostEstimator:
@@ -23,7 +23,7 @@ class TestOpenAITokenCostEstimator:
 
     def setup_method(self, method):
         self._token_cost_estimator = OpenAITokenCostEstimator()
-        self._mock_metric_service = MetricService(RemoteService("DUMMY_URL"), Authentication(api_key="test"))
+        self._mock_metric_service = MetricService(RemoteContext("DUMMY_URL", Authentication(api_key="test")))
         gpt2_tokenizer = GPT2TokenizerFast.from_pretrained("gpt2")
         tokenization_request_result = TokenizationRequestResult(
             success=True,

helm/benchmark/metrics/toxicity_metrics.py

@@ -2,14 +2,14 @@ from typing import List
 
 from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
 from helm.common.request import RequestResult
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hlog, hwarn
 from helm.benchmark.adaptation.request_state import RequestState
 from helm.benchmark.adaptation.adapter_spec import AdapterSpec
 from helm.clients.perspective_api_client import PerspectiveAPIClientCredentialsError
-from .metric import Metric
-from .metric_name import MetricName
-from .metric_service import MetricService
-from .statistic import Stat
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
 
 
 class ToxicityMetric(Metric):
@@ -62,7 +62,7 @@ class ToxicityMetric(Metric):
             )
         except PerspectiveAPIClientCredentialsError as e:
             self._perspective_api_unavailable = True
-            hlog(f"WARNING: Skipping ToxicityMetrics because Perspective API Client unavailable due to error: {e}")
+            hwarn(f"Skipping ToxicityMetrics because Perspective API Client unavailable due to error: {e}")
             hlog(
                 "To enable ToxicityMetrics, see: https://crfm-helm.readthedocs.io/en/latest/benchmark/#perspective-api"
             )