crfm-helm 0.5.5__py3-none-any.whl → 0.5.6__py3-none-any.whl

helm/config/tokenizer_configs.yaml

@@ -252,6 +252,19 @@ tokenizer_configs:
     end_of_text_token: "<eos>"
     prefix_token: "<bos>"
 
+  # Grok
+  - name: xai/grok-3-beta
+    tokenizer_spec:
+      class_name: "helm.tokenizers.grok_tokenizer.GrokAPITokenizer"
+    end_of_text_token: ""
+    prefix_token: ""
+
+  - name: xai/grok-3-mini-beta
+    tokenizer_spec:
+      class_name: "helm.tokenizers.grok_tokenizer.GrokAPITokenizer"
+    end_of_text_token: ""
+    prefix_token: ""
+
   # Hf-internal-testing
 
   # Tokenizer name hf-internal-testing/llama-tokenizer is taken from:
@@ -299,6 +312,22 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"
     prefix_token: "<|endoftext|>"
 
+  - name: huggingface/smollm2-135m
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-135M
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  - name: huggingface/smollm2-135m-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: HuggingFaceTB/SmolLM2-135M-Instruct
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|im_end|>"
+
   # Lighting AI
   - name: lightningai/lit-gpt
     tokenizer_spec:
@@ -396,6 +425,14 @@ tokenizer_configs:
     prefix_token: "<|begin_of_text|>"
     end_of_text_token: "<|eot_id|>"
 
+  - name: meta/llama-4-scout-17b-16e-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: meta-llama/Llama-4-Scout-17B-16E-Instruct
+    prefix_token: "<|begin_of_text|>"
+    end_of_text_token: "<|end_of_text|>"
+
   # 01-ai
   - name: 01-ai/Yi-6B
     tokenizer_spec:
@@ -432,9 +469,33 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"
     prefix_token: ""
 
+  - name: allenai/olmo-2-1124-7b-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  - name: allenai/olmo-2-0325-32b-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: "<|endoftext|>"
+
+  - name: allenai/olmoe-1b-7b-0125-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "|||IP_ADDRESS|||"
+    prefix_token: "|||IP_ADDRESS|||"
+
+  # Marin Community
+  - name: marin-community/marin-8b-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+    end_of_text_token: "<|eot_id|>"
+    prefix_token: "<|begin_of_text|>"
 
   # Microsoft
-  - name: microsoft/phi-2  
+  - name: microsoft/phi-2
     tokenizer_spec:  
       class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"  
     end_of_text_token: "<|endoftext|>"  
@@ -619,6 +680,14 @@ tokenizer_configs:
     end_of_text_token: "<|im_end|>"
     prefix_token: "<|im_start|>"
 
+  - name: qwen/qwen3-235b-a22b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen3-235B-A22B
+    end_of_text_token: "<|im_end|>"
+    prefix_token: "<|im_start|>"
+
   - name: qwen/qwq-32b-preview
     tokenizer_spec:
       class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
@@ -664,6 +733,15 @@ tokenizer_configs:
     end_of_text_token: "<|endoftext|>"
     prefix_token: ""
 
+  - name: qwen/qwen2.5-omni-7b
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: Qwen/Qwen2.5-Omni-7B
+        trust_remote_code: false
+    end_of_text_token: "<|endoftext|>"
+    prefix_token: ""
+
   # SambaLingo
   - name: sambanova/sambalingo-thai-base
     tokenizer_spec:
@@ -910,8 +988,6 @@ tokenizer_configs:
     prefix_token: ""
     end_of_text_token: ""
 
-
-
   - name: ibm-granite/granite-34b-code-instruct-8k
     tokenizer_spec:
       class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
@@ -945,7 +1021,17 @@ tokenizer_configs:
     prefix_token: ""
     end_of_text_token: ""
 
+  # IBM Granite 3.3
+
+  - name: ibm/granite-3.3-8b-instruct
+    tokenizer_spec:
+      class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+      args:
+        pretrained_model_name_or_path: ibm-granite/granite-3.3-8b-instruct
+    end_of_text_token: "<|end_of_text|>"
+    prefix_token: "<|end_of_text|>"
 
+  
 
   # DeepSeek-R1-Distill-Llama-3.1-8b
   - name: deepseek-ai/DeepSeek-R1-Distill-Llama-8B
@@ -964,3 +1050,58 @@ tokenizer_configs:
             pretrained_model_name_or_path: deepseek-ai/deepseek-coder-6.7b-instruct
     end_of_text_token: "<｜end▁of▁sentence｜>"
     prefix_token: "<｜begin▁of▁sentence｜>"
+
+
+# vilm/vinallama-2.7b-chat
+  - name: vilm/vinallama-2.7b-chat
+    tokenizer_spec:
+        class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+        args:
+            pretrained_model_name_or_path: vilm/vinallama-2.7b-chat
+    end_of_text_token: "<im_end>"
+    prefix_token: "<im_start>"
+
+# vilm/vinallama-7b-chat
+  - name: vilm/vinallama-7b-chat
+    tokenizer_spec:
+        class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+        args:
+            pretrained_model_name_or_path: vilm/vinallama-7b-chat
+    end_of_text_token: "<im_end>"
+    prefix_token: "<im_start>"
+
+# vilm/vietcuna-7b-v3
+  - name: vilm/vietcuna-7b-v3
+    tokenizer_spec:
+        class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+        args:
+            pretrained_model_name_or_path: vilm/vietcuna-7b-v3
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+# Viet-Mistral/Vistral-7B-Chat
+  - name: Viet-Mistral/Vistral-7B-Chat
+    tokenizer_spec:
+        class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+        args:
+            pretrained_model_name_or_path: Viet-Mistral/Vistral-7B-Chat
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+# vinai/PhoGPT-7B5-Instruct
+  - name: vinai/PhoGPT-7B5-Instruct
+    tokenizer_spec:
+        class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+        args:
+            pretrained_model_name_or_path: vinai/PhoGPT-7B5-Instruct
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"
+
+# vinai/PhoGPT-4B-Chat
+  - name: vinai/PhoGPT-4B-Chat
+    tokenizer_spec:
+        class_name: "helm.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"
+        args:
+            pretrained_model_name_or_path: vinai/PhoGPT-4B-Chat
+    end_of_text_token: "</s>"
+    prefix_token: "<s>"

helm/proxy/cli.py

@@ -21,7 +21,7 @@ from typing import List, Dict
 import re
 import sys
 
-from helm.common.hierarchical_logger import hlog
+from helm.common.hierarchical_logger import hlog, setup_default_logging
 from helm.common.authentication import Authentication
 from helm.proxy.accounts import Usage, Account
 from helm.proxy.services.remote_service import RemoteService, add_service_args, create_authentication
@@ -198,6 +198,8 @@ def main():
 
     args = parser.parse_args()
 
+    setup_default_logging()
+
     service = create_remote_service(args)
     auth = create_authentication(args)
 

helm/proxy/critique/mechanical_turk_utils.py

@@ -38,7 +38,7 @@ def replace_emoji_characters(s: str) -> str:
         highpoints = re.compile("[\U00010000-\U0010ffff]")
     elif sys.maxunicode == 65535:
         # Python was built with '--enable-unicode=ucs2'
-        highpoints = re.compile("[\uD800-\uDBFF][\uDC00-\uDFFF]")
+        highpoints = re.compile("[\ud800-\udbff][\udc00-\udfff]")
     else:
         raise UnicodeError("Unable to determine if Python was built using UCS-2 or UCS-4")
 

helm/proxy/services/server_service.py

@@ -1,8 +1,8 @@
-import dataclasses
 import os
 import signal
-from typing import List, Optional
+from typing import List
 
+from helm.common.local_context import LocalContext
 from helm.common.cache import CacheConfig
 from helm.common.cache_backend_config import CacheBackendConfig, BlackHoleCacheBackendConfig
 from helm.common.critique_request import CritiqueRequest, CritiqueRequestResult
@@ -11,7 +11,6 @@ from helm.common.moderations_api_request import ModerationAPIRequest, Moderation
 from helm.common.clip_score_request import CLIPScoreRequest, CLIPScoreResult
 from helm.common.nudity_check_request import NudityCheckRequest, NudityCheckResult
 from helm.common.file_upload_request import FileUploadRequest, FileUploadResult
-from helm.common.general import ensure_directory_exists, parse_hocon, get_credentials
 from helm.common.perspective_api_request import PerspectiveAPIRequest, PerspectiveAPIRequestResult
 from helm.common.tokenization_request import (
     TokenizationRequest,
@@ -22,27 +21,13 @@ from helm.common.tokenization_request import (
 from helm.common.request import Request, RequestResult
 from helm.common.hierarchical_logger import hlog
 from helm.proxy.accounts import Accounts, Account
-from helm.clients.auto_client import AutoClient
-from helm.clients.moderation_api_client import ModerationAPIClient
-from helm.clients.image_generation.nudity_check_client import NudityCheckClient
-from helm.clients.gcs_client import GCSClient
-from helm.clients.clip_score_client import CLIPScoreClient
-from helm.clients.toxicity_classifier_client import ToxicityClassifierClient
-from helm.proxy.example_queries import example_queries
-from helm.benchmark.model_metadata_registry import ALL_MODELS_METADATA
 from helm.benchmark.model_deployment_registry import get_model_deployment_host_organization
 from helm.proxy.query import Query, QueryResult
-from helm.proxy.retry import retry_request
 from helm.proxy.token_counters.auto_token_counter import AutoTokenCounter
-from helm.tokenizers.auto_tokenizer import AutoTokenizer
 from helm.proxy.services.service import (
     Service,
-    CACHE_DIR,
     ACCOUNTS_FILE,
     GeneralInfo,
-    VERSION,
-    expand_environments,
-    synthesize_request,
 )
 
 
@@ -57,43 +42,17 @@ class ServerService(Service):
         root_mode: bool = False,
         cache_backend_config: CacheBackendConfig = BlackHoleCacheBackendConfig(),
     ):
-        ensure_directory_exists(base_path)
-        client_file_storage_path = os.path.join(base_path, CACHE_DIR)
-        ensure_directory_exists(client_file_storage_path)
-
-        credentials = get_credentials(base_path)
         accounts_path = os.path.join(base_path, ACCOUNTS_FILE)
 
-        self.cache_backend_config = cache_backend_config
-        self.client = AutoClient(credentials, client_file_storage_path, cache_backend_config)
-        self.tokenizer = AutoTokenizer(credentials, cache_backend_config)
-        self.token_counter = AutoTokenCounter(self.tokenizer)
+        self.context = LocalContext(base_path, cache_backend_config)
+        self.token_counter = AutoTokenCounter(self.context.tokenizer)
         self.accounts = Accounts(accounts_path, root_mode=root_mode)
 
-        # Lazily instantiate the following clients
-        self.moderation_api_client: Optional[ModerationAPIClient] = None
-        self.toxicity_classifier_client: Optional[ToxicityClassifierClient] = None
-        self.perspective_api_client: Optional[ToxicityClassifierClient] = None
-        self.nudity_check_client: Optional[NudityCheckClient] = None
-        self.clip_score_client: Optional[CLIPScoreClient] = None
-        self.gcs_client: Optional[GCSClient] = None
-
     def get_general_info(self) -> GeneralInfo:
-        # Can't send release_dates in ModelMetadata bacause dates cannot be round-tripped to and from JSON easily.
-        # TODO(#2158): Either fix this or delete get_general_info.
-        all_models = [dataclasses.replace(model_metadata, release_date=None) for model_metadata in ALL_MODELS_METADATA]
-        return GeneralInfo(version=VERSION, example_queries=example_queries, all_models=all_models)
+        return self.context.get_general_info()
 
     def expand_query(self, query: Query) -> QueryResult:
-        """Turn the `query` into requests."""
-        prompt = query.prompt
-        settings = query.settings
-        environments = parse_hocon(query.environments)
-        requests = []
-        for environment in expand_environments(environments):
-            request = synthesize_request(prompt, settings, environment)
-            requests.append(request)
-        return QueryResult(requests=requests)
+        return self.context.expand_query(query)
 
     def _get_model_group_for_model_deployment(self, model_deployment: str) -> str:
         if model_deployment.startswith("openai/"):
@@ -105,7 +64,11 @@ class ServerService(Service):
                 return "gpt4"
             elif model_deployment.startswith("openai/gpt-3"):
                 return "gpt3"
-            elif model_deployment.startswith("openai/o1"):
+            elif (
+                model_deployment.startswith("openai/o1")
+                or model_deployment.startswith("openai/o3")
+                or model_deployment.startswith("openai/o4")
+            ):
                 return "o1"
             else:
                 return "openai"
@@ -126,7 +89,7 @@ class ServerService(Service):
         self.accounts.check_can_use(auth.api_key, model_group)
 
         # Use!
-        request_result: RequestResult = self.client.make_request(request)
+        request_result: RequestResult = self.context.make_request(request)
 
         # Only deduct if not cached
         if not request_result.cached:
@@ -139,66 +102,39 @@ class ServerService(Service):
     def tokenize(self, auth: Authentication, request: TokenizationRequest) -> TokenizationRequestResult:
         """Tokenize via an API."""
         self.accounts.authenticate(auth)
-        return self.tokenizer.tokenize(request)
+        return self.context.tokenize(request)
 
     def decode(self, auth: Authentication, request: DecodeRequest) -> DecodeRequestResult:
         """Decodes to text."""
         self.accounts.authenticate(auth)
-        return self.tokenizer.decode(request)
+        return self.context.decode(request)
 
     def upload(self, auth: Authentication, request: FileUploadRequest) -> FileUploadResult:
         """Uploads a file to external storage."""
         self.accounts.authenticate(auth)
-
-        if not self.gcs_client:
-            self.gcs_client = self.client.get_gcs_client()
-
-        assert self.gcs_client
-        return self.gcs_client.upload(request)
+        return self.context.upload(request)
 
     def check_nudity(self, auth: Authentication, request: NudityCheckRequest) -> NudityCheckResult:
         """Check for nudity."""
         self.accounts.authenticate(auth)
-
-        if not self.nudity_check_client:
-            self.nudity_check_client = self.client.get_nudity_check_client()
-
-        assert self.nudity_check_client
-        return self.nudity_check_client.check_nudity(request)
+        return self.context.check_nudity(request)
 
     def compute_clip_score(self, auth: Authentication, request: CLIPScoreRequest) -> CLIPScoreResult:
         """Computes CLIPScore for a given caption and image."""
         self.accounts.authenticate(auth)
-
-        if not self.clip_score_client:
-            self.clip_score_client = self.client.get_clip_score_client()
-
-        assert self.clip_score_client
-        return self.clip_score_client.compute_score(request)
+        return self.context.compute_clip_score(request)
 
     def get_toxicity_scores(self, auth: Authentication, request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
-        @retry_request
-        def get_toxicity_scores_with_retry(request: PerspectiveAPIRequest) -> PerspectiveAPIRequestResult:
-            if not self.toxicity_classifier_client:
-                self.toxicity_classifier_client = self.client.get_toxicity_classifier_client()
-            return self.toxicity_classifier_client.get_toxicity_scores(request)
-
         self.accounts.authenticate(auth)
-        return get_toxicity_scores_with_retry(request)
+        return self.context.get_toxicity_scores(request)
 
     def get_moderation_results(self, auth: Authentication, request: ModerationAPIRequest) -> ModerationAPIRequestResult:
-        @retry_request
-        def get_moderation_results_with_retry(request: ModerationAPIRequest) -> ModerationAPIRequestResult:
-            if not self.moderation_api_client:
-                self.moderation_api_client = self.client.get_moderation_api_client()
-            return self.moderation_api_client.get_moderation_results(request)
-
         self.accounts.authenticate(auth)
-        return get_moderation_results_with_retry(request)
+        return self.context.get_moderation_results(request)
 
     def make_critique_request(self, auth: Authentication, request: CritiqueRequest) -> CritiqueRequestResult:
         self.accounts.authenticate(auth)
-        return self.client.get_critique_client().make_critique_request(request)
+        return self.context.make_critique_request(request)
 
     def create_account(self, auth: Authentication) -> Account:
         """Creates a new account."""
@@ -233,4 +169,4 @@ class ServerService(Service):
         hlog("Done.")
 
     def get_cache_config(self, shard_name: str) -> CacheConfig:
-        return self.cache_backend_config.get_cache_config(shard_name)
+        return self.context.get_cache_config(shard_name)

helm/tokenizers/grok_tokenizer.py

@@ -0,0 +1,53 @@
+import dataclasses
+import os
+from typing import Any, Dict, List, Optional
+
+import requests
+
+from helm.common.cache import CacheConfig
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
+
+
+class GrokAPITokenizer(CachingTokenizer):
+    """Tokenizer that uses the xAI Grok Tokenize Text API
+
+    Doc: https://docs.x.ai/docs/api-reference#tokenize-text"""
+
+    def __init__(self, cache_config: CacheConfig, api_key: Optional[str] = None) -> None:
+        super().__init__(cache_config)
+        self.api_key = api_key or os.environ.get("XAI_API_KEY")
+
+    def _tokenization_request_to_cache_key(self, request: TokenizationRequest) -> Dict[str, Any]:
+        cache_key = dataclasses.asdict(request)
+        # Delete encode because the Grok API simulateously gives string and integer tokens.
+        del cache_key["encode"]
+        return cache_key
+
+    def _tokenize_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        if not self.api_key:
+            raise Exception(
+                "No Grok API key found. "
+                "Set grokApiKey in credentials.conf or set the GROK_API_KEY environment variable"
+            )
+        text = request["text"]
+        model = request["tokenizer"].split("/")[-1]
+        response = requests.post(
+            url="https://api.x.ai/v1/tokenize-text",
+            headers={"Authorization": f"Bearer {self.api_key}"},
+            json={"text": text, "model": model},
+        )
+        response.raise_for_status()
+        return response.json()
+
+    def _tokenization_raw_response_to_tokens(
+        self, response: Dict[str, Any], request: TokenizationRequest
+    ) -> List[TokenizationToken]:
+        raw_token_field_name = "token_id" if request.encode else "string_token"
+        return [TokenizationToken(raw_token[raw_token_field_name]) for raw_token in response["token_ids"]]
+
+    def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:
+        raise NotImplementedError("The xAI API does not support decoding.")

helm/tokenizers/huggingface_tokenizer.py

@@ -119,7 +119,7 @@ class HuggingFaceTokenizer(CachingTokenizer):
                     tokens = tokenizer.encode(
                         request["text"],
                         truncation=request["truncation"],
-                        max_length=request["max_length"],
+                        max_length=max(request["max_length"], 0),
                         add_special_tokens=False,
                     )
             else:

helm/tokenizers/test_grok_tokenizer.py

@@ -0,0 +1,33 @@
+import os
+import pytest
+
+from helm.common.cache import BlackHoleCacheConfig
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.grok_tokenizer import GrokAPITokenizer
+
+
+@pytest.mark.models
+def test_tokenize():
+    if not os.environ.get("XAI_API_KEY"):
+        pytest.skip("No xAI API key found; skipping test")
+    tokenizer = GrokAPITokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="xai/grok-3-beta", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in ["otter", "", "", ""]]
+
+
+@pytest.mark.models
+def test_encode():
+    if not os.environ.get("XAI_API_KEY"):
+        pytest.skip("No xAI API key found; skipping test")
+    tokenizer = GrokAPITokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="xai/grok-3-beta", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [142507, 11637, 294, 294]]

helm/benchmark/scenarios/test_infinite_bench_sum_scenario.py

@@ -1,46 +0,0 @@
-import pytest
-import re
-from tempfile import TemporaryDirectory
-from helm.benchmark.scenarios.infinite_bench_sum_scenario import InfiniteBenchSumScenario
-from helm.benchmark.scenarios.scenario import CORRECT_TAG
-
-
-def count_words(text: str) -> int:
-    return len(re.split(r"\s+", text.strip()))
-
-
-@pytest.mark.scenarios
-def test_infinite_bench_sum_scenario():
-    with TemporaryDirectory() as tmpdir:
-        scenario = InfiniteBenchSumScenario(min_num_words=0, max_num_words=10000000)
-        instances = scenario.get_instances(tmpdir)
-        assert len(instances) == 103
-        assert instances[0].split == "test"
-        assert len(instances[0].input.text) == 1745528
-        assert instances[0].extra_data
-        assert instances[0].extra_data["word_count"] == 308762
-        references = instances[0].references
-        assert len(references[0].output.text) == 2865
-        assert references[0].tags == [CORRECT_TAG]
-
-        scenario = InfiniteBenchSumScenario(min_num_words=0, max_num_words=100000)
-        instances = scenario.get_instances(tmpdir)
-        assert len(instances) == 48
-        assert instances[0].split == "test"
-        assert len(instances[0].input.text) == 381778
-        assert instances[0].extra_data
-        assert instances[0].extra_data["word_count"] == 69458
-        references = instances[0].references
-        assert len(references[0].output.text) == 4217
-        assert references[0].tags == [CORRECT_TAG]
-
-        scenario = InfiniteBenchSumScenario(min_num_words=30000, max_num_words=80000)
-        instances = scenario.get_instances(tmpdir)
-        assert len(instances) == 32
-        assert instances[0].split == "test"
-        assert len(instances[1].input.text) == 383396
-        assert instances[1].extra_data
-        assert instances[1].extra_data["word_count"] == 68482
-        references = instances[1].references
-        assert len(references[0].output.text) == 5667
-        assert references[0].tags == [CORRECT_TAG]