crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/benchmark/augmentations/person_name_perturbation.py

@@ -9,8 +9,9 @@ from typing import Dict, List, Optional, Set
 
 from helm.benchmark.scenarios.scenario import Input, Instance, Reference, Output
 from helm.common.general import ensure_file_downloaded, ensure_directory_exists, match_case
-from .perturbation_description import PerturbationDescription
-from .perturbation import Perturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import Perturbation
+from helm.benchmark.runner import get_benchmark_output_path
 
 
 # Pull this out so serialization works for multiprocessing
@@ -35,7 +36,7 @@ class PersonNamePerturbation(Perturbation):
         "https://storage.googleapis.com/crfm-helm-public/source_datasets/"
         "augmentations/person_name_perturbation/person_names.txt"
     )
-    OUTPUT_PATH = os.path.join("benchmark_output", "perturbations", name)
+    OUTPUT_PATH = os.path.join(get_benchmark_output_path(), "perturbations", name)
 
     """ Name types """
     FIRST_NAME = "first_name"
@@ -153,8 +154,6 @@ class PersonNamePerturbation(Perturbation):
                 find the gender association for a source_word, we randomly
                 pick from one of the target names.
         """
-        # TODO: Update path so it is not hard-coded to benchmark_output
-        # https://github.com/stanford-crfm/benchmarking/issues/493
         self.output_path: str = self.OUTPUT_PATH
         Path(self.output_path).mkdir(parents=True, exist_ok=True)
 

helm/benchmark/augmentations/perturbation.py

@@ -4,13 +4,12 @@ from random import Random
 from typing import List, Optional
 
 
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 from helm.benchmark.scenarios.scenario import Input, Instance, Reference, Output
 from helm.common.object_spec import ObjectSpec, create_object
 
 
 class Perturbation(ABC):
-
     # Unique name to describe perturbation
     name: str
 
@@ -49,18 +48,41 @@ class TextPerturbation(Perturbation, ABC):
 
         description = replace(self.description, seed=seed)
 
+        perturbed_input: Input
+        if instance.input.multimedia_content:
+            perturbed_media_objects = []
+            for media_object in instance.input.multimedia_content.media_objects:
+                # Apply perturbations to the text data of the multimedia content
+                if media_object.is_type("text") and media_object.text is not None:
+                    perturbed_media_objects.append(replace(media_object, text=self.perturb(media_object.text, rng)))
+                else:
+                    perturbed_media_objects.append(media_object)
+
+            perturbed_input = Input(
+                multimedia_content=replace(instance.input.multimedia_content, media_objects=perturbed_media_objects)
+            )
+        else:
+            perturbed_input = Input(text=self.perturb(instance.input.text, rng))
+
         # Don't modify `id` of `Instance` here.
         # All the perturbed Instances generated from a single Instance should have the same ID.
         return replace(
             instance,
-            input=Input(text=self.perturb(instance.input.text, rng)),
+            input=perturbed_input,
             references=references,
             perturbation=description,
+            contrast_inputs=[instance.input],
         )
 
     def _perturb_reference(self, reference: Reference, rng: Random) -> Reference:
         """Generates a new Reference by perturbing the output and tagging the Reference."""
-        return replace(reference, output=Output(text=self.perturb(reference.output.text, rng)), tags=reference.tags)
+        return replace(
+            reference,
+            output=Output(
+                text=self.perturb(reference.output.text, rng), multimedia_content=reference.output.multimedia_content
+            ),
+            tags=reference.tags,
+        )
 
     @abstractmethod
     def perturb(self, text: str, rng: Random) -> str:

helm/benchmark/augmentations/perturbation_description.py

@@ -23,7 +23,7 @@ class PerturbationDescription:
     computed_on: str = PERTURBATION_PERTURBED
     """Which types of Instances we are evaluating, to be populated during metric evaluation. PERTURBATION_PERTURBED
     (default) means we are evaluating on perturbed instances, PERTURBATION_ORIGINAL means we are evaluating the
-    unperturbed version of instances where this perturbation appplies, and, PERTURBATION_WORST means the the minimum
+    unperturbed version of instances where this perturbation applies, and, PERTURBATION_WORST means the the minimum
     metric between the two."""
 
     seed: Optional[int] = None

helm/benchmark/augmentations/space_perturbation.py

@@ -2,8 +2,8 @@ from dataclasses import dataclass
 from random import Random
 import re
 
-from .perturbation import TextPerturbation
-from .perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
 
 
 class SpacePerturbation(TextPerturbation):

helm/benchmark/augmentations/suffix_perturbation.py

@@ -0,0 +1,29 @@
+from dataclasses import dataclass
+from random import Random
+
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+
+
+class SuffixPerturbation(TextPerturbation):
+    """
+    Appends a suffix to the end of the text. Example:
+
+    A picture of a dog -> A picture of a dog, picasso
+    """
+
+    @dataclass(frozen=True)
+    class Description(PerturbationDescription):
+        suffix: str = ""
+
+    name: str = "style"
+
+    def __init__(self, suffix: str):
+        self._suffix: str = suffix
+
+    @property
+    def description(self) -> PerturbationDescription:
+        return SuffixPerturbation.Description(name=self.name, suffix=self._suffix)
+
+    def perturb(self, text: str, rng: Random) -> str:
+        return f"{text}, {self._suffix}"

helm/benchmark/augmentations/synonym_perturbation.py

@@ -10,8 +10,9 @@ from nltk.corpus import wordnet
 import spacy
 
 from helm.common.general import match_case, ensure_file_downloaded
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.runner import get_benchmark_output_path
 
 
 class SynonymPerturbation(TextPerturbation):
@@ -57,7 +58,7 @@ class SynonymPerturbation(TextPerturbation):
             spacy.cli.download("en_core_web_sm")  # type: ignore
             self.spacy_model = spacy.load("en_core_web_sm")
 
-        output_dir = os.path.join("benchmark_output", "perturbations", self.name)
+        output_dir = os.path.join(get_benchmark_output_path(), "perturbations", self.name)
         Path(output_dir).mkdir(parents=True, exist_ok=True)
         nltk.data.path.append(output_dir)
         try:

helm/benchmark/augmentations/test_perturbation.py

@@ -2,19 +2,24 @@
 from typing import List
 import unittest
 
+from helm.common.media_object import MediaObject, MultimediaObject
 from helm.benchmark.scenarios.scenario import Input, Instance, Output, Reference
-from .data_augmenter import DataAugmenter
-from .extra_space_perturbation import ExtraSpacePerturbation
-from .misspelling_perturbation import MisspellingPerturbation
-from .contraction_expansion_perturbation import ContractionPerturbation, ExpansionPerturbation
-from .typos_perturbation import TyposPerturbation
-from .filler_words_perturbation import FillerWordsPerturbation
-from .synonym_perturbation import SynonymPerturbation
-from .lowercase_perturbation import LowerCasePerturbation
-from .space_perturbation import SpacePerturbation
-from .dialect_perturbation import DialectPerturbation
-from .person_name_perturbation import PersonNamePerturbation
-from .gender_perturbation import GenderPerturbation
+from helm.benchmark.augmentations.data_augmenter import DataAugmenter
+from helm.benchmark.augmentations.extra_space_perturbation import ExtraSpacePerturbation
+from helm.benchmark.augmentations.misspelling_perturbation import MisspellingPerturbation
+from helm.benchmark.augmentations.contraction_expansion_perturbation import (
+    ContractionPerturbation,
+    ExpansionPerturbation,
+)
+from helm.benchmark.augmentations.typos_perturbation import TyposPerturbation
+from helm.benchmark.augmentations.filler_words_perturbation import FillerWordsPerturbation
+from helm.benchmark.augmentations.synonym_perturbation import SynonymPerturbation
+from helm.benchmark.augmentations.lowercase_perturbation import LowerCasePerturbation
+from helm.benchmark.augmentations.space_perturbation import SpacePerturbation
+from helm.benchmark.augmentations.dialect_perturbation import DialectPerturbation
+from helm.benchmark.augmentations.person_name_perturbation import PersonNamePerturbation
+from helm.benchmark.augmentations.gender_perturbation import GenderPerturbation
+from helm.benchmark.augmentations.suffix_perturbation import SuffixPerturbation
 
 
 def test_extra_space_perturbation():
@@ -32,6 +37,35 @@ def test_extra_space_perturbation():
     assert instances[1].references[0].output.text == "some name"
 
 
+def test_multimodal_text_perturbation():
+    data_augmenter = DataAugmenter(perturbations=[ExtraSpacePerturbation(num_spaces=3)])
+    input: Input = Input(
+        multimedia_content=MultimediaObject(
+            [
+                MediaObject(text="Hello what is", content_type="text/plain"),
+                MediaObject(text="your name", content_type="text/plain"),
+            ]
+        )
+    )
+    instance: Instance = Instance(id="id0", input=input, references=[Reference(Output(text="some name"), tags=[])])
+    instances: List[Instance] = data_augmenter.generate([instance], include_original=True)
+
+    assert len(instances) == 2
+
+    # Test that the first instance is unperturbed
+    assert instances[0].id == "id0"
+    assert instances[0].perturbation is None
+    media_objects = instances[0].input.multimedia_content.media_objects
+    assert media_objects[0].text == "Hello what is"
+    assert media_objects[1].text == "your name"
+
+    assert instances[1].id == "id0"
+    assert instances[1].perturbation.name == "extra_space"
+    media_objects = instances[1].input.multimedia_content.media_objects
+    assert media_objects[0].text == "Hello   what   is"
+    assert media_objects[1].text == "your   name"
+
+
 def test_misspelling_perturbation():
     data_augmenter = DataAugmenter(perturbations=[MisspellingPerturbation(prob=1.0)])
     instance: Instance = Instance(
@@ -145,7 +179,6 @@ def test_space_perturbation():
     instance: Instance = Instance(id="id0", input=Input(text="Hello World!\nQuite a day, huh?"), references=[])
     instances: List[Instance] = data_augmenter.generate([instance], include_original=True)
 
-    print(instances)
     assert len(instances) == 2
     assert instances[1].perturbation.name == "space"
     assert instances[1].input.text == "Hello   World!\nQuite a  day,   huh?"
@@ -162,7 +195,6 @@ def test_dialect_perturbation():
     )
     instances: List[Instance] = data_augmenter.generate([instance], include_original=True)
 
-    print(instances)
     assert len(instances) == 2
     assert instances[1].perturbation.name == "dialect"
     assert instances[1].input.text == "I gon remember dis day to b the best day of mah life."
@@ -188,7 +220,6 @@ def test_person_name_perturbation():
     )
     instances: List[Instance] = data_augmenter.generate([instance], include_original=True)
 
-    print(instances)
     assert len(instances) == 2
     assert instances[1].perturbation.name == "person_name"
     assert (
@@ -209,7 +240,6 @@ def test_gender_pronoun_perturbation():
     )
     instances: List[Instance] = data_augmenter.generate([instance], include_original=True)
 
-    print(instances)
     assert len(instances) == 2
     assert instances[1].perturbation.mode == "pronouns"
     assert instances[1].input.text == "Did she mention that she was coming with her parents and their friends?"
@@ -227,13 +257,22 @@ def test_gender_term_perturbation():
     )
     instances: List[Instance] = data_augmenter.generate([instance], include_original=True)
 
-    print(instances)
     assert len(instances) == 2
     assert instances[1].perturbation.mode == "terms"
     assert instances[1].input.text == "His granddaughters looked a lot like their mom."
     assert instances[1].references[0].output.text == "How did their mother look like?"
 
 
+def test_suffix_perturbation():
+    data_augmenter = DataAugmenter(perturbations=[SuffixPerturbation(suffix="pixel art")])
+    instance: Instance = Instance(id="id0", input=Input(text="A blue dog"), references=[])
+    instances: List[Instance] = data_augmenter.generate([instance], include_original=True)
+
+    assert len(instances) == 2
+    assert instances[1].perturbation.suffix == "pixel art"
+    assert instances[1].input.text == "A blue dog, pixel art"
+
+
 # TODO(#1958) Fix the logic to renable this test
 @unittest.skip("Currently cannot replace words at either text boundary.")
 def test_gender_term_perturbation_edge_word():
@@ -247,7 +286,6 @@ def test_gender_term_perturbation_edge_word():
     )
     instances: List[Instance] = data_augmenter.generate([instance], include_original=False)
 
-    print(instances)
     assert len(instances) == 1
     assert instances[0].input.text == "mom said it is okay"
     assert instances[0].references[0].output.text == "Sure he did daughter"
@@ -266,6 +304,5 @@ def test_gender_term_perturbation_consequtive_words():
     )
     instances: List[Instance] = data_augmenter.generate([instance], include_original=False)
 
-    print(instances)
     assert len(instances) == 1
     assert instances[0].input.text == "I'm a mom mom: my daughter has a daughter."

helm/benchmark/augmentations/translate_perturbation.py

@@ -0,0 +1,31 @@
+from dataclasses import dataclass
+from random import Random
+
+from helm.clients.google_translate_client import GoogleTranslateClient
+from helm.benchmark.augmentations.perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+
+
+class TranslatePerturbation(TextPerturbation):
+    """
+    Translates to different languages.
+    """
+
+    @dataclass(frozen=True)
+    class Description(PerturbationDescription):
+        # Language code to translate to. Needs a default value since we inherit from `PerturbationDescription`
+        language_code: str = "zh-CN"
+
+    name: str = "translate"
+    should_perturb_references: bool = True
+
+    def __init__(self, language_code: str):
+        self.language_code: str = language_code
+        self.google_translate_client = GoogleTranslateClient()
+
+    @property
+    def description(self) -> PerturbationDescription:
+        return TranslatePerturbation.Description(name=self.name, language_code=self.language_code)
+
+    def perturb(self, text: str, rng: Random) -> str:
+        return self.google_translate_client.translate(text, self.language_code)

helm/benchmark/augmentations/typos_perturbation.py

@@ -1,8 +1,8 @@
 from dataclasses import dataclass
 from random import Random
 
-from .perturbation_description import PerturbationDescription
-from .perturbation import TextPerturbation
+from helm.benchmark.augmentations.perturbation_description import PerturbationDescription
+from helm.benchmark.augmentations.perturbation import TextPerturbation
 
 
 class TyposPerturbation(TextPerturbation):

helm/benchmark/config_registry.py

@@ -4,16 +4,18 @@ import importlib_resources as resources
 from helm.benchmark.model_deployment_registry import register_model_deployments_from_path
 from helm.benchmark.model_metadata_registry import register_model_metadata_from_path
 from helm.benchmark.tokenizer_config_registry import register_tokenizer_configs_from_path
+from helm.benchmark.runner_config_registry import register_runner_config_from_path
 
 
 MODEL_METADATA_FILE: str = "model_metadata.yaml"
 TOKENIZER_CONFIGS_FILE: str = "tokenizer_configs.yaml"
 MODEL_DEPLOYMENTS_FILE: str = "model_deployments.yaml"
+RUNNER_CONFIG_FILE: str = "runner_config.yaml"
 
 CONFIG_PACKAGE = "helm.config"
 
 
-def register_configs_from_directory(dir_path) -> None:
+def register_configs_from_directory(dir_path: str) -> None:
     model_metadata_path = os.path.join(dir_path, MODEL_METADATA_FILE)
     if os.path.isfile(model_metadata_path):
         register_model_metadata_from_path(model_metadata_path)
@@ -26,6 +28,10 @@ def register_configs_from_directory(dir_path) -> None:
     if os.path.isfile(model_deployments_path):
         register_model_deployments_from_path(model_deployments_path)
 
+    runner_config_path = os.path.join(dir_path, RUNNER_CONFIG_FILE)
+    if os.path.isfile(runner_config_path):
+        register_runner_config_from_path(runner_config_path)
+
 
 def register_builtin_configs_from_helm_package() -> None:
     package_path = str(resources.files(CONFIG_PACKAGE))

helm/benchmark/data_preprocessor.py

@@ -1,7 +1,7 @@
 from typing import List
 from helm.common.hierarchical_logger import htrack
-from .augmentations.data_augmenter import create_data_augmenter, DataAugmenterSpec, DataAugmenter
-from .scenarios.scenario import Instance, TRAIN_SPLIT, EVAL_SPLITS
+from helm.benchmark.augmentations.data_augmenter import create_data_augmenter, DataAugmenterSpec, DataAugmenter
+from helm.benchmark.scenarios.scenario import Instance, TRAIN_SPLIT, EVAL_SPLITS
 
 
 class DataPreprocessor:

helm/benchmark/executor.py

@@ -1,13 +1,19 @@
 from typing import Optional
 from dataclasses import dataclass, replace
 
+from helm.common.context import Context
+from helm.common.local_context import LocalContext
+from helm.common.remote_context import RemoteContext
+from helm.common.cache_backend_config import (
+    CacheBackendConfig,
+    BlackHoleCacheBackendConfig,
+    MongoCacheBackendConfig,
+    SqliteCacheBackendConfig,
+)
 from helm.common.general import parallel_map
-from helm.common.hierarchical_logger import htrack, hlog
-from helm.common.request import RequestResult, Sequence
+from helm.common.hierarchical_logger import htrack, hlog, hwarn
+from helm.common.request import RequestResult, GeneratedOutput
 from helm.common.authentication import Authentication
-from helm.proxy.services.remote_service import RemoteService
-from helm.proxy.services.server_service import ServerService
-from helm.proxy.services.service import Service
 from helm.benchmark.adaptation.scenario_state import ScenarioState
 from helm.benchmark.adaptation.request_state import RequestState
 
@@ -18,28 +24,36 @@ class ExecutorError(Exception):
 
 @dataclass(frozen=True)
 class ExecutionSpec:
-    # If non-empty, URL of the proxy server we send requests to (e.g., http://localhost:1959).
+
     url: Optional[str]
+    """If non-empty, URL of the proxy server we send requests to (e.g., http://localhost:1959)."""
 
-    # Pass into the service
     auth: Authentication
+    """Authentication that will be passed into the remote service, if using the remote context."""
 
-    # Path where API credentials and cache is stored.
-    # This path is the same as `--base-path` when launching the proxy server (see server.py).
-    # Required when url is not set.
     local_path: Optional[str]
+    """Path where API credentials and cache is stored.
+
+    This path is the same as `--base-path` when launching the proxy server (see server.py).
+    Required when url is not set."""
 
-    # How many threads to have at once
     parallelism: int
+    """How many threads to have at once"""
 
-    # Whether to skip execution
     dry_run: bool = False
+    """Whether to skip execution"""
+
+    sqlite_cache_backend_config: Optional[SqliteCacheBackendConfig] = None
+    """If set, SQLite will be used for the cache.
+
+    This specifies the directory in which the SQLite cache will store files.
+    At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set."""
 
-    # URL to the MongoDB database.
-    # If non-empty, the MongoDB database will be used for caching instead of SQLite.
-    # Example format: mongodb://[username:password@]host1[:port1]/[dbname]
-    # For full format, see: https://www.mongodb.com/docs/manual/reference/connection-string/
-    mongo_uri: str = ""
+    mongo_cache_backend_config: Optional[MongoCacheBackendConfig] = None
+    """If set, MongoDB will be used for the cache.
+
+    This specifies the MongoDB database to be used by the MongoDB cache.
+    At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set."""
 
 
 class Executor:
@@ -51,14 +65,25 @@ class Executor:
     def __init__(self, execution_spec: ExecutionSpec):
         self.execution_spec = execution_spec
 
-        self.service: Service
+        cache_backend_config: CacheBackendConfig
+        if execution_spec.sqlite_cache_backend_config and execution_spec.mongo_cache_backend_config:
+            raise ExecutorError("At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set.")
+        elif execution_spec.sqlite_cache_backend_config:
+            cache_backend_config = execution_spec.sqlite_cache_backend_config
+        elif execution_spec.mongo_cache_backend_config:
+            cache_backend_config = execution_spec.mongo_cache_backend_config
+        else:
+            cache_backend_config = BlackHoleCacheBackendConfig()
+
+        self.context: Context
         if execution_spec.url:
             hlog(f"Running using remote API proxy server: {execution_spec.url}")
-            self.service = RemoteService(execution_spec.url)
+            self.context = RemoteContext(execution_spec.url, execution_spec.auth)
         elif execution_spec.local_path:
             hlog(f"Running in local mode with base path: {execution_spec.local_path}")
-            self.service = ServerService(
-                base_path=execution_spec.local_path, root_mode=True, mongo_uri=execution_spec.mongo_uri
+            self.context = LocalContext(
+                base_path=execution_spec.local_path,
+                cache_backend_config=cache_backend_config,
             )
         else:
             raise ValueError("Either the proxy server URL or the local path must be set")
@@ -77,17 +102,21 @@ class Executor:
         )
 
         hlog(f"Processed {len(request_states)} requests")
-        return ScenarioState(scenario_state.adapter_spec, request_states)
+        return ScenarioState(
+            adapter_spec=scenario_state.adapter_spec,
+            request_states=request_states,
+            annotator_specs=scenario_state.annotator_specs,
+        )
 
     def process(self, state: RequestState) -> RequestState:
         try:
-            result: RequestResult = self.service.make_request(self.execution_spec.auth, state.request)
+            result: RequestResult = self.context.make_request(state.request)
         except Exception as e:
             raise ExecutorError(f"{str(e)} Request: {state.request}") from e
         if not result.success:
             if result.error_flags and not result.error_flags.is_fatal:
-                hlog(f"WARNING: Non-fatal error treated as empty completion: {result.error}")
-                result.completions = [Sequence(text="", logprob=0, tokens=[])]
+                hwarn(f"Non-fatal error treated as empty completion: {result.error}")
+                result.completions = [GeneratedOutput(text="", logprob=0, tokens=[])]
             else:
                 raise ExecutorError(f"{str(result.error)} Request: {state.request}")
         return replace(state, result=result)

helm/benchmark/huggingface_registration.py

@@ -1,10 +1,9 @@
 import os
-from typing import Optional
+from typing import Optional, Dict, Union
 
 from helm.benchmark.model_deployment_registry import (
     ClientSpec,
     ModelDeployment,
-    WindowServiceSpec,
     register_model_deployment,
 )
 from helm.benchmark.model_metadata_registry import (
@@ -14,27 +13,44 @@ from helm.benchmark.model_metadata_registry import (
 )
 from helm.benchmark.tokenizer_config_registry import TokenizerConfig, TokenizerSpec, register_tokenizer_config
 from helm.common.hierarchical_logger import hlog
+from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
 
 
 def register_huggingface_model(
-    helm_model_name: str, pretrained_model_name_or_path: str, revision: Optional[str] = None
+    helm_model_name: str,
+    pretrained_model_name_or_path: str,
+    revision: Optional[str] = None,
 ) -> None:
-    object_spec_args = {"pretrained_model_name_or_path": pretrained_model_name_or_path}
+    object_spec_args: Dict[str, Union[str, bool]] = {"pretrained_model_name_or_path": pretrained_model_name_or_path}
     if revision:
         object_spec_args["revision"] = revision
 
+    # Auto-infer model properties from the tokenizer.
+    create_tokenizer_args: Dict[str, str] = {"pretrained_model_name_or_path": pretrained_model_name_or_path}
+    if revision:
+        create_tokenizer_args["revision"] = revision
+    with HuggingFaceTokenizer.create_tokenizer(**create_tokenizer_args) as tokenizer:
+        max_sequence_length = tokenizer.model_max_length
+        end_of_text_token = tokenizer.eos_token or ""
+        prefix_token = tokenizer.bos_token or ""
+    # If the tokenizer config has a model_max_length of 1000000000000000019884624838656
+    # it means that model creator did not specify model_max_length.
+    if max_sequence_length > 1_000_000:
+        raise ValueError(
+            f"Could not infer the model_max_length of Hugging Face model {pretrained_model_name_or_path}, so "
+            f"--enable-huggingface-models and --enable-local-huggingface-models cannot be used for this model. "
+            f"Please configure the model using prod_env/model_deployments.yaml instead."
+        )
+
     model_deployment = ModelDeployment(
         name=helm_model_name,
         client_spec=ClientSpec(
-            class_name="helm.proxy.clients.huggingface_client.HuggingFaceClient",
+            class_name="helm.clients.huggingface_client.HuggingFaceClient",
             args=object_spec_args,
         ),
         model_name=helm_model_name,
         tokenizer_name=helm_model_name,
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.huggingface_window_service.HuggingFaceWindowService",
-            args=object_spec_args,
-        ),
+        max_sequence_length=max_sequence_length,
     )
 
     # We check if the model is already registered because we don't want to
@@ -51,9 +67,11 @@ def register_huggingface_model(
     tokenizer_config = TokenizerConfig(
         name=helm_model_name,
         tokenizer_spec=TokenizerSpec(
-            class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer",
+            class_name="helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer",
             args=object_spec_args,
         ),
+        end_of_text_token=end_of_text_token,
+        prefix_token=prefix_token,
     )
     register_tokenizer_config(tokenizer_config)