crfm-helm 0.4.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/benchmark/executor.py

@@ -1,9 +1,15 @@
 from typing import Optional
 from dataclasses import dataclass, replace
+from helm.common.cache_backend_config import (
+    CacheBackendConfig,
+    BlackHoleCacheBackendConfig,
+    MongoCacheBackendConfig,
+    SqliteCacheBackendConfig,
+)
 
 from helm.common.general import parallel_map
 from helm.common.hierarchical_logger import htrack, hlog
-from helm.common.request import RequestResult, Sequence
+from helm.common.request import RequestResult, GeneratedOutput
 from helm.common.authentication import Authentication
 from helm.proxy.services.remote_service import RemoteService
 from helm.proxy.services.server_service import ServerService
@@ -18,28 +24,36 @@ class ExecutorError(Exception):
 
 @dataclass(frozen=True)
 class ExecutionSpec:
-    # If non-empty, URL of the proxy server we send requests to (e.g., http://localhost:1959).
+
     url: Optional[str]
+    """If non-empty, URL of the proxy server we send requests to (e.g., http://localhost:1959)."""
 
-    # Pass into the service
     auth: Authentication
+    """Authentication that will be passed into the local service, if using the local service."""
 
-    # Path where API credentials and cache is stored.
-    # This path is the same as `--base-path` when launching the proxy server (see server.py).
-    # Required when url is not set.
     local_path: Optional[str]
+    """Path where API credentials and cache is stored.
+
+    This path is the same as `--base-path` when launching the proxy server (see server.py).
+    Required when url is not set."""
 
-    # How many threads to have at once
     parallelism: int
+    """How many threads to have at once"""
 
-    # Whether to skip execution
     dry_run: bool = False
+    """Whether to skip execution"""
+
+    sqlite_cache_backend_config: Optional[SqliteCacheBackendConfig] = None
+    """If set, SQLite will be used for the cache.
+
+    This specifies the directory in which the SQLite cache will store files.
+    At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set."""
 
-    # URL to the MongoDB database.
-    # If non-empty, the MongoDB database will be used for caching instead of SQLite.
-    # Example format: mongodb://[username:password@]host1[:port1]/[dbname]
-    # For full format, see: https://www.mongodb.com/docs/manual/reference/connection-string/
-    mongo_uri: str = ""
+    mongo_cache_backend_config: Optional[MongoCacheBackendConfig] = None
+    """If set, MongoDB will be used for the cache.
+
+    This specifies the MongoDB database to be used by the MongoDB cache.
+    At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set."""
 
 
 class Executor:
@@ -51,6 +65,16 @@ class Executor:
     def __init__(self, execution_spec: ExecutionSpec):
         self.execution_spec = execution_spec
 
+        cache_backend_config: CacheBackendConfig
+        if execution_spec.sqlite_cache_backend_config and execution_spec.mongo_cache_backend_config:
+            raise ExecutorError("At most one of sqlite_cache_backend_config and mongo_cache_backend_config can be set.")
+        elif execution_spec.sqlite_cache_backend_config:
+            cache_backend_config = execution_spec.sqlite_cache_backend_config
+        elif execution_spec.mongo_cache_backend_config:
+            cache_backend_config = execution_spec.mongo_cache_backend_config
+        else:
+            cache_backend_config = BlackHoleCacheBackendConfig()
+
         self.service: Service
         if execution_spec.url:
             hlog(f"Running using remote API proxy server: {execution_spec.url}")
@@ -58,7 +82,9 @@ class Executor:
         elif execution_spec.local_path:
             hlog(f"Running in local mode with base path: {execution_spec.local_path}")
             self.service = ServerService(
-                base_path=execution_spec.local_path, root_mode=True, mongo_uri=execution_spec.mongo_uri
+                base_path=execution_spec.local_path,
+                root_mode=True,
+                cache_backend_config=cache_backend_config,
             )
         else:
             raise ValueError("Either the proxy server URL or the local path must be set")
@@ -77,7 +103,11 @@ class Executor:
         )
 
         hlog(f"Processed {len(request_states)} requests")
-        return ScenarioState(scenario_state.adapter_spec, request_states)
+        return ScenarioState(
+            adapter_spec=scenario_state.adapter_spec,
+            request_states=request_states,
+            annotator_specs=scenario_state.annotator_specs,
+        )
 
     def process(self, state: RequestState) -> RequestState:
         try:
@@ -87,7 +117,7 @@ class Executor:
         if not result.success:
             if result.error_flags and not result.error_flags.is_fatal:
                 hlog(f"WARNING: Non-fatal error treated as empty completion: {result.error}")
-                result.completions = [Sequence(text="", logprob=0, tokens=[])]
+                result.completions = [GeneratedOutput(text="", logprob=0, tokens=[])]
             else:
                 raise ExecutorError(f"{str(result.error)} Request: {state.request}")
         return replace(state, result=result)

helm/benchmark/huggingface_registration.py

@@ -4,7 +4,6 @@ from typing import Optional
 from helm.benchmark.model_deployment_registry import (
     ClientSpec,
     ModelDeployment,
-    WindowServiceSpec,
     register_model_deployment,
 )
 from helm.benchmark.model_metadata_registry import (
@@ -14,6 +13,7 @@ from helm.benchmark.model_metadata_registry import (
 )
 from helm.benchmark.tokenizer_config_registry import TokenizerConfig, TokenizerSpec, register_tokenizer_config
 from helm.common.hierarchical_logger import hlog
+from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
 
 
 def register_huggingface_model(
@@ -23,18 +23,29 @@ def register_huggingface_model(
     if revision:
         object_spec_args["revision"] = revision
 
+    # Auto-infer model properties from the tokenizer.
+    with HuggingFaceTokenizer.create_tokenizer(**object_spec_args) as tokenizer:
+        max_sequence_length = tokenizer.model_max_length
+        end_of_text_token = tokenizer.eos_token or ""
+        prefix_token = tokenizer.bos_token or ""
+    # If the tokenizer config has a model_max_length of 1000000000000000019884624838656
+    # it means that model creator did not specify model_max_length.
+    if max_sequence_length > 1_000_000:
+        raise ValueError(
+            f"Could not infer the model_max_length of Hugging Face model {pretrained_model_name_or_path}, so "
+            f"--enable-huggingface-models and --enable-local-huggingface-models cannot be used for this model. "
+            f"Please configure the model using prod_env/model_deployments.yaml instead."
+        )
+
     model_deployment = ModelDeployment(
         name=helm_model_name,
         client_spec=ClientSpec(
-            class_name="helm.proxy.clients.huggingface_client.HuggingFaceClient",
+            class_name="helm.clients.huggingface_client.HuggingFaceClient",
             args=object_spec_args,
         ),
         model_name=helm_model_name,
         tokenizer_name=helm_model_name,
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.huggingface_window_service.HuggingFaceWindowService",
-            args=object_spec_args,
-        ),
+        max_sequence_length=max_sequence_length,
     )
 
     # We check if the model is already registered because we don't want to
@@ -51,9 +62,11 @@ def register_huggingface_model(
     tokenizer_config = TokenizerConfig(
         name=helm_model_name,
         tokenizer_spec=TokenizerSpec(
-            class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer",
+            class_name="helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer",
             args=object_spec_args,
         ),
+        end_of_text_token=end_of_text_token,
+        prefix_token=prefix_token,
     )
     register_tokenizer_config(tokenizer_config)