crfm-helm 0.5.4__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/benchmark/augmentations/contrast_sets_perturbation.py

@@ -4,8 +4,8 @@ from random import Random
 from typing import List, Optional
 
 from helm.benchmark.scenarios.scenario import Instance, Reference, Input
-from .perturbation_description import PerturbationDescription
-from .perturbation import Perturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import Perturbation
 
 
 class ContrastSetsPerturbation(Perturbation):

helm/benchmark/augmentations/dialect_perturbation.py

@@ -7,8 +7,9 @@ from pathlib import Path
 from typing import Dict, Optional, List
 
 from helm.common.general import match_case, ensure_file_downloaded
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.runner import get_benchmark_output_path
 
 
 class DialectPerturbation(TextPerturbation):
@@ -20,7 +21,7 @@ class DialectPerturbation(TextPerturbation):
     should_perturb_references: bool = True
 
     """ Output path to store external files and folders """
-    OUTPUT_PATH = os.path.join("benchmark_output", "perturbations", name)
+    OUTPUT_PATH = os.path.join(get_benchmark_output_path(), "perturbations", name)
 
     """ Dictionary mapping dialects to one another """
     SAE = "SAE"
@@ -74,8 +75,6 @@ class DialectPerturbation(TextPerturbation):
                 self.MAPPING_DICTS for the provided source and target classes
                 will be used, if available.
         """
-        # TODO: Update path so it is not hard-coded to benchmark_output
-        # https://github.com/stanford-crfm/benchmarking/issues/493
         self.output_path: str = self.OUTPUT_PATH
         Path(self.output_path).mkdir(parents=True, exist_ok=True)
 

helm/benchmark/augmentations/extra_space_perturbation.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from random import Random
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class ExtraSpacePerturbation(TextPerturbation):

helm/benchmark/augmentations/filler_words_perturbation.py

@@ -1,7 +1,7 @@
 from dataclasses import dataclass
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 from random import Random
 

helm/benchmark/augmentations/gender_perturbation.py

@@ -5,8 +5,8 @@ import re
 from typing import Dict, List, Optional, Tuple
 
 from helm.common.general import match_case
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
 
 
 """ Gender term mappings """

helm/benchmark/augmentations/lowercase_perturbation.py

@@ -1,7 +1,7 @@
 from random import Random
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class LowerCasePerturbation(TextPerturbation):

helm/benchmark/augmentations/mild_mix_perturbation.py

@@ -1,11 +1,11 @@
 from random import Random
 
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
-from .lowercase_perturbation import LowerCasePerturbation
-from .contraction_expansion_perturbation import ContractionPerturbation
-from .space_perturbation import SpacePerturbation
-from .misspelling_perturbation import MisspellingPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.lowercase_perturbation import LowerCasePerturbation
+from helm.benchmark.augmentations.contraction_expansion_perturbation import ContractionPerturbation
+from helm.benchmark.augmentations.space_perturbation import SpacePerturbation
+from helm.benchmark.augmentations.misspelling_perturbation import MisspellingPerturbation
 
 
 class MildMixPerturbation(TextPerturbation):

helm/benchmark/augmentations/misspelling_perturbation.py

@@ -6,8 +6,8 @@ from random import Random
 from typing import Dict, List
 
 from helm.common.general import match_case
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 # The implementation below is based on the following list of common misspellings:

helm/benchmark/augmentations/person_name_perturbation.py

@@ -9,8 +9,9 @@ from typing import Dict, List, Optional, Set
 
 from helm.benchmark.scenarios.scenario import Input, Instance, Reference, Output
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists, match_case
-from .perturbation_description import PerturbationDescription
-from .perturbation import Perturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import Perturbation
+from helm.benchmark.runner import get_benchmark_output_path
 
 
 # Pull this out so serialization works for multiprocessing
@@ -35,7 +36,7 @@ class PersonNamePerturbation(Perturbation):
         "https://storage.googleapis.com/crfm-helm-public/source_datasets/"
         "augmentations/person_name_perturbation/person_names.txt"
     )
-    OUTPUT_PATH = os.path.join("benchmark_output", "perturbations", name)
+    OUTPUT_PATH = os.path.join(get_benchmark_output_path(), "perturbations", name)
 
     """ Name types """
     FIRST_NAME = "first_name"
@@ -153,8 +154,6 @@ class PersonNamePerturbation(Perturbation):
                 find the gender association for a source_word, we randomly
                 pick from one of the target names.
         """
-        # TODO: Update path so it is not hard-coded to benchmark_output
-        # https://github.com/stanford-crfm/benchmarking/issues/493
         self.output_path: str = self.OUTPUT_PATH
         Path(self.output_path).mkdir(parents=True, exist_ok=True)
 

helm/benchmark/augmentations/perturbation.py

@@ -4,7 +4,7 @@ from random import Random
 from typing import List, Optional
 
 
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 from helm.benchmark.scenarios.scenario import Input, Instance, Reference, Output
 from helm.common.object_spec import ObjectSpec, create_object
 

helm/benchmark/augmentations/space_perturbation.py

@@ -2,8 +2,8 @@ from dataclasses import dataclass
 from random import Random
 import re
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class SpacePerturbation(TextPerturbation):

helm/benchmark/augmentations/suffix_perturbation.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from random import Random
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class SuffixPerturbation(TextPerturbation):

helm/benchmark/augmentations/synonym_perturbation.py

@@ -10,8 +10,9 @@ from nltk.corpus import wordnet
 import spacy
 
 from helm.common.general import match_case, ensure_file_downloaded
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.runner import get_benchmark_output_path
 
 
 class SynonymPerturbation(TextPerturbation):
@@ -57,7 +58,7 @@ class SynonymPerturbation(TextPerturbation):
             spacy.cli.download("en_core_web_sm")  # type: ignore
             self.spacy_model = spacy.load("en_core_web_sm")
 
-        output_dir = os.path.join("benchmark_output", "perturbations", self.name)
+        output_dir = os.path.join(get_benchmark_output_path(), "perturbations", self.name)
         Path(output_dir).mkdir(parents=True, exist_ok=True)
         nltk.data.path.append(output_dir)
         try:

helm/benchmark/augmentations/test_perturbation.py

@@ -4,19 +4,22 @@ import unittest
 
 from helm.common.media_object import MediaObject, MultimediaObject
 from helm.benchmark.scenarios.scenario import Input, Instance, Output, Reference
-from .data_augmenter import DataAugmenter
-from .extra_space_perturbation import ExtraSpacePerturbation
-from .misspelling_perturbation import MisspellingPerturbation
-from .contraction_expansion_perturbation import ContractionPerturbation, ExpansionPerturbation
-from .typos_perturbation import TyposPerturbation
-from .filler_words_perturbation import FillerWordsPerturbation
-from .synonym_perturbation import SynonymPerturbation
-from .lowercase_perturbation import LowerCasePerturbation
-from .space_perturbation import SpacePerturbation
-from .dialect_perturbation import DialectPerturbation
-from .person_name_perturbation import PersonNamePerturbation
-from .gender_perturbation import GenderPerturbation
-from .suffix_perturbation import SuffixPerturbation
+from helm.benchmark.augmentations.data_augmenter import DataAugmenter
+from helm.benchmark.augmentations.extra_space_perturbation import ExtraSpacePerturbation
+from helm.benchmark.augmentations.misspelling_perturbation import MisspellingPerturbation
+from helm.benchmark.augmentations.contraction_expansion_perturbation import (
+    ContractionPerturbation,
+    ExpansionPerturbation,
+)
+from helm.benchmark.augmentations.typos_perturbation import TyposPerturbation
+from helm.benchmark.augmentations.filler_words_perturbation import FillerWordsPerturbation
+from helm.benchmark.augmentations.synonym_perturbation import SynonymPerturbation
+from helm.benchmark.augmentations.lowercase_perturbation import LowerCasePerturbation
+from helm.benchmark.augmentations.space_perturbation import SpacePerturbation
+from helm.benchmark.augmentations.dialect_perturbation import DialectPerturbation
+from helm.benchmark.augmentations.person_name_perturbation import PersonNamePerturbation
+from helm.benchmark.augmentations.gender_perturbation import GenderPerturbation
+from helm.benchmark.augmentations.suffix_perturbation import SuffixPerturbation
 
 
 def test_extra_space_perturbation():

helm/benchmark/augmentations/translate_perturbation.py

@@ -2,8 +2,8 @@ from dataclasses import dataclass
 from random import Random
 
 from helm.clients.google_translate_client import GoogleTranslateClient
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class TranslatePerturbation(TextPerturbation):

helm/benchmark/augmentations/typos_perturbation.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from random import Random
 
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
 
 
 class TyposPerturbation(TextPerturbation):

helm/benchmark/data_preprocessor.py

@@ -1,7 +1,7 @@
 from typing import List
 from helm.common.hierarchical_logger import htrack
-from .augmentations.data_augmenter import create_data_augmenter, DataAugmenterSpec, DataAugmenter
-from .scenarios.scenario import Instance, TRAIN_SPLIT, EVAL_SPLITS
+from helm.benchmark.augmentations.data_augmenter import create_data_augmenter, DataAugmenterSpec, DataAugmenter
+from helm.benchmark.scenarios.scenario import Instance, TRAIN_SPLIT, EVAL_SPLITS
 
 
 class DataPreprocessor:

helm/benchmark/executor.py

@@ -1,19 +1,19 @@
 from typing import Optional
 from dataclasses import dataclass, replace
+
+from helm.common.context import Context
+from helm.common.local_context import LocalContext
+from helm.common.remote_context import RemoteContext
 from helm.common.cache_backend_config import (
     CacheBackendConfig,
     BlackHoleCacheBackendConfig,
     MongoCacheBackendConfig,
     SqliteCacheBackendConfig,
 )
-
 from helm.common.general import parallel_map
-from helm.common.hierarchical_logger import htrack, hlog
+from helm.common.hierarchical_logger import htrack, hlog, hwarn
 from helm.common.request import RequestResult, GeneratedOutput
 from helm.common.authentication import Authentication
-from helm.proxy.services.remote_service import RemoteService
-from helm.proxy.services.server_service import ServerService
-from helm.proxy.services.service import Service
 from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.adaptation.request_state import RequestState
 
@@ -29,7 +29,7 @@ class ExecutionSpec:
     """If non-empty, URL of the proxy server we send requests to (e.g., http://localhost:1959)."""
 
     auth: Authentication
-    """Authentication that will be passed into the local service, if using the local service."""
+    """Authentication that will be passed into the remote service, if using the remote context."""
 
     local_path: Optional[str]
     """Path where API credentials and cache is stored.
@@ -75,15 +75,14 @@ class Executor:
         else:
             cache_backend_config = BlackHoleCacheBackendConfig()
 
-        self.service: Service
+        self.context: Context
         if execution_spec.url:
             hlog(f"Running using remote API proxy server: {execution_spec.url}")
-            self.service = RemoteService(execution_spec.url)
+            self.context = RemoteContext(execution_spec.url, execution_spec.auth)
         elif execution_spec.local_path:
             hlog(f"Running in local mode with base path: {execution_spec.local_path}")
-            self.service = ServerService(
+            self.context = LocalContext(
                 base_path=execution_spec.local_path,
-                root_mode=True,
                 cache_backend_config=cache_backend_config,
             )
         else:
@@ -111,12 +110,12 @@ class Executor:
 
     def process(self, state: RequestState) -> RequestState:
         try:
-            result: RequestResult = self.service.make_request(self.execution_spec.auth, state.request)
+            result: RequestResult = self.context.make_request(state.request)
         except Exception as e:
             raise ExecutorError(f"{str(e)} Request: {state.request}") from e
         if not result.success:
             if result.error_flags and not result.error_flags.is_fatal:
-                hlog(f"WARNING: Non-fatal error treated as empty completion: {result.error}")
+                hwarn(f"Non-fatal error treated as empty completion: {result.error}")
                 result.completions = [GeneratedOutput(text="", logprob=0, tokens=[])]
             else:
                 raise ExecutorError(f"{str(result.error)} Request: {state.request}")

helm/benchmark/huggingface_registration.py

@@ -20,13 +20,10 @@ def register_huggingface_model(
     helm_model_name: str,
     pretrained_model_name_or_path: str,
     revision: Optional[str] = None,
-    openvino: Optional[bool] = False,
 ) -> None:
     object_spec_args: Dict[str, Union[str, bool]] = {"pretrained_model_name_or_path": pretrained_model_name_or_path}
     if revision:
         object_spec_args["revision"] = revision
-    if openvino:
-        object_spec_args["openvino"] = openvino
 
     # Auto-infer model properties from the tokenizer.
     create_tokenizer_args: Dict[str, str] = {"pretrained_model_name_or_path": pretrained_model_name_or_path}
@@ -79,7 +76,7 @@ def register_huggingface_model(
     register_tokenizer_config(tokenizer_config)
 
 
-def register_huggingface_hub_model_from_flag_value(raw_model_string: str, openvino=False) -> None:
+def register_huggingface_hub_model_from_flag_value(raw_model_string: str) -> None:
     raw_model_string_parts = raw_model_string.split("@")
     pretrained_model_name_or_path: str
     revision: Optional[str]
@@ -96,11 +93,10 @@ def register_huggingface_hub_model_from_flag_value(raw_model_string: str, openvi
         helm_model_name=raw_model_string,
         pretrained_model_name_or_path=pretrained_model_name_or_path,
         revision=revision,
-        openvino=openvino,
     )
 
 
-def register_huggingface_local_model_from_flag_value(path: str, openvino=False) -> None:
+def register_huggingface_local_model_from_flag_value(path: str) -> None:
     if not path:
         raise ValueError("Path to Hugging Face model must be non-empty")
     path_parts = os.path.split(path)
@@ -108,5 +104,4 @@ def register_huggingface_local_model_from_flag_value(path: str, openvino=False)
     register_huggingface_model(
         helm_model_name=helm_model_name,
         pretrained_model_name_or_path=path,
-        openvino=openvino,
     )

helm/benchmark/metrics/aci_bench_metrics.py

@@ -0,0 +1,14 @@
+from helm.benchmark.annotation.aci_bench_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
+
+
+class ACIBenchMetric(LLMJuryMetric):
+    """Score metrics for ACIBench."""
+
+    def __init__(self):
+        super().__init__(
+            metric_name="aci_bench_accuracy",
+            scenario_name="aci_bench",
+            annotator_models=ANNOTATOR_MODELS,
+            default_score=1.0,
+        )

helm/benchmark/metrics/basic_metrics.py

@@ -5,8 +5,8 @@ from typing import List, Dict, Set
 from urllib.parse import unquote
 
 import numpy as np
-import scipy
-import calibration as cal
+import scipy  # type: ignore
+import calibration as cal  # type: ignore
 from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.metrics.evaluate_reference_metrics import compute_reference_metrics
 from helm.benchmark.metrics.efficiency_metrics import EfficiencyMetric
@@ -25,10 +25,10 @@ from helm.benchmark.window_services.window_service import WindowService
 from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
 from helm.benchmark.window_services.tokenizer_service import TokenizerService
 from helm.benchmark.scenarios.scenario import CORRECT_TAG, Instance
-from .metric import Metric, MetricInterface, MetricResult, add_context, get_unique_stat_by_name
-from .metric_name import MetricContext, MetricName
-from .metric_service import MetricService
-from .statistic import Stat, merge_stat
+from helm.benchmark.metrics.metric import Metric, MetricInterface, MetricResult, add_context, get_unique_stat_by_name
+from helm.benchmark.metrics.metric_name import MetricContext, MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat, merge_stat
 
 
 def get_num_bytes(tokens: List[Token]) -> int:

helm/benchmark/metrics/bbq_metrics.py

@@ -3,8 +3,8 @@ from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMe
 
 from helm.common.request import RequestResult
 from helm.benchmark.adaptation.request_state import RequestState
-from .metric_name import MetricName
-from .statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.statistic import Stat
 
 AMBIGUOUS_TAG = "ambiguous"
 NON_AMBIGUOUS_TAG = "non-ambiguous"

helm/benchmark/metrics/bias_metrics.py

@@ -8,9 +8,18 @@ from helm.benchmark.metrics.evaluate_instances_metric import EvaluateInstancesMe
 
 from helm.common.request import RequestResult, GeneratedOutput
 from helm.benchmark.adaptation.request_state import RequestState
-from .statistic import Stat
-from .metric_name import MetricName
-from .bias_word_lists import GENDER_TO_WORD_LISTS, RACE_TO_NAME_LISTS, ADJECTIVE_LIST, PROFESSION_LIST
+from helm.benchmark.metrics.nltk_helper import install_nltk_resources
+from helm.benchmark.metrics.statistic import Stat
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.bias_word_lists import (
+    GENDER_TO_WORD_LISTS,
+    RACE_TO_NAME_LISTS,
+    ADJECTIVE_LIST,
+    PROFESSION_LIST,
+)
+
+
+install_nltk_resources()
 
 
 class BiasMetric(EvaluateInstancesMetric):

helm/benchmark/metrics/bias_word_lists.py

@@ -1,4 +1,4 @@
-""" Bias words utilised to compute the bias metrics.
+"""Bias words utilised to compute the bias metrics.
 
 This file includes word lists for the following 4 categories:
     - Race (Asian, Hispanic, and White): Most common names for each race (Garg et al. 2018)

helm/benchmark/metrics/bigcodebench_metrics.py

@@ -0,0 +1,25 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class BigCodeBenchMetric(Metric):
+    """Score metrics for BigCodeBench."""
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        assert request_state.annotations
+        score = request_state.annotations["bigcodebench"]["pass_at_one"]
+        return [
+            Stat(MetricName("bigcodebench_p@1")).add(score),
+        ]

helm/benchmark/metrics/bird_sql_metrics.py

@@ -0,0 +1,28 @@
+from typing import List
+
+from helm.benchmark.adaptation.adapter_spec import AdapterSpec
+from helm.benchmark.adaptation.request_state import RequestState
+from helm.benchmark.metrics.metric import Metric
+from helm.benchmark.metrics.metric_name import MetricName
+from helm.benchmark.metrics.metric_service import MetricService
+from helm.benchmark.metrics.statistic import Stat
+
+
+class BirdSQLMetric(Metric):
+    """Score metrics for Bird-SQL."""
+
+    ANNOTATOR_NAME = "bird_sql"
+
+    def evaluate_generation(
+        self,
+        adapter_spec: AdapterSpec,
+        request_state: RequestState,
+        metric_service: MetricService,
+        eval_cache_path: str,
+    ) -> List[Stat]:
+        if not request_state.annotations:
+            raise Exception("Request state did not have annotations.")
+        predicted_result = request_state.annotations[self.ANNOTATOR_NAME]["predicted_result"]
+        ground_truth_result = request_state.annotations[self.ANNOTATOR_NAME]["ground_truth_result"]
+        execution_accuracy = int(set(predicted_result) == set(ground_truth_result))
+        return [Stat(MetricName("execution_accuracy")).add(execution_accuracy)]

helm/benchmark/metrics/chw_care_plan_metrics.py

@@ -0,0 +1,14 @@
+from helm.benchmark.annotation.chw_care_plan_annotator import ANNOTATOR_MODELS
+from helm.benchmark.metrics.llm_jury_metrics import LLMJuryMetric
+
+
+class CHWCarePlanMetric(LLMJuryMetric):
+    """Score metrics for CHWCarePlan."""
+
+    def __init__(self):
+        super().__init__(
+            metric_name="chw_care_plan_accuracy",
+            scenario_name="chw_care_plan",
+            annotator_models=ANNOTATOR_MODELS,
+            default_score=1.0,
+        )