crfm-helm 0.4.0__py3-none-any.whl → 0.5.10__py3-none-any.whl

helm/{proxy/tokenizers → tokenizers}/lit_gpt_tokenizer.py

@@ -5,7 +5,7 @@ import torch
 
 from helm.common.cache import CacheConfig
 from helm.common.optional_dependencies import OptionalDependencyNotInstalled
-from .caching_tokenizer import CachingTokenizer
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
 
 try:
     from lit_gpt import Tokenizer as InternalTokenizer

helm/tokenizers/simple_tokenizer.py

@@ -0,0 +1,33 @@
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    DecodeRequestResult,
+    TokenizationRequest,
+    TokenizationRequestResult,
+    TokenizationToken,
+)
+from helm.tokenizers.tokenizer import Tokenizer
+
+
+class SimpleTokenizer(Tokenizer):
+    """Simple tokenizer for tutorials and for debugging."""
+
+    def tokenize(self, request: TokenizationRequest) -> TokenizationRequestResult:
+        if request.encode:
+            return TokenizationRequestResult(
+                success=True,
+                cached=False,
+                tokens=[TokenizationToken(ord(character)) for character in request.text],
+                text=request.text,
+            )
+        else:
+            return TokenizationRequestResult(
+                success=True,
+                cached=False,
+                tokens=[TokenizationToken(character) for character in request.text],
+                text=request.text,
+            )
+
+    def decode(self, request: DecodeRequest) -> DecodeRequestResult:
+        return DecodeRequestResult(
+            success=True, cached=False, text="".join([chr(code_point) for code_point in request.tokens])
+        )

helm/tokenizers/test_ai21_tokenizer.py

@@ -0,0 +1,48 @@
+import pytest
+
+from helm.common.cache import BlackHoleCacheConfig
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    TokenizationRequest,
+    TokenizationToken,
+)
+
+
+@pytest.mark.models
+def test_tokenize():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="ai21/jamba-instruct-tokenizer", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [
+        TokenizationToken(token) for token in ["ot", "ter", "▁", "<0xF0>", "<0x9F>", "<0xA6>", "<0xA6>"]
+    ]
+
+
+@pytest.mark.models
+def test_encode():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="ai21/jamba-instruct-tokenizer", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [1860, 1901, 62934, 1784, 1703, 1710, 1710]]
+
+
+@pytest.mark.models
+def test_decode():
+    from helm.tokenizers.ai21_tokenizer import AI21LocalTokenizer
+
+    tokenizer = AI21LocalTokenizer(cache_config=BlackHoleCacheConfig())
+    request = DecodeRequest(
+        tokenizer="ai21/jamba-instruct-tokenizer", tokens=[1860, 1901, 62934, 1784, 1703, 1710, 1710]
+    )
+    result = tokenizer.decode(request)
+    assert result.success
+    assert not result.cached
+    assert result.text == "otter 🦦"

helm/{proxy/tokenizers → tokenizers}/test_anthropic_tokenizer.py

@@ -10,7 +10,7 @@ from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from helm.proxy.tokenizers.anthropic_tokenizer import AnthropicTokenizer
+from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
 
 
 class TestAnthropicTokenizer:
@@ -21,7 +21,11 @@ class TestAnthropicTokenizer:
     def setup_method(self, method):
         cache_file = tempfile.NamedTemporaryFile(delete=False)
         self.cache_path: str = cache_file.name
-        self.tokenizer = AnthropicTokenizer(SqliteCacheConfig(self.cache_path))
+        self.tokenizer = HuggingFaceTokenizer(
+            SqliteCacheConfig(self.cache_path),
+            tokenizer_name="anthropic/claude",
+            pretrained_model_name_or_path="Xenova/claude-tokenizer",
+        )
 
     def teardown_method(self, method):
         os.remove(self.cache_path)

helm/tokenizers/test_cohere_tokenizer.py

@@ -0,0 +1,39 @@
+import pytest
+
+from helm.common.cache import BlackHoleCacheConfig
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.cohere_tokenizer import CohereLocalTokenizer
+
+
+@pytest.mark.models
+def test_tokenize():
+    tokenizer = CohereLocalTokenizer(api_key=None, cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="cohere/command", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in ["ot", "ter", " �", "�", "�"]]
+
+
+@pytest.mark.models
+def test_encode():
+    tokenizer = CohereLocalTokenizer(api_key=None, cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="cohere/command", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [1741, 1779, 7728, 107, 107]]
+
+
+@pytest.mark.models
+def test_decode():
+    tokenizer = CohereLocalTokenizer(api_key=None, cache_config=BlackHoleCacheConfig())
+    request = DecodeRequest(tokenizer="cohere/command", tokens=[1741, 1779, 7728, 107, 107])
+    result = tokenizer.decode(request)
+    assert result.success
+    assert not result.cached
+    assert result.text == "otter 🦦"

helm/tokenizers/test_grok_tokenizer.py

@@ -0,0 +1,33 @@
+import os
+import pytest
+
+from helm.common.cache import BlackHoleCacheConfig
+from helm.common.tokenization_request import (
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.grok_tokenizer import GrokAPITokenizer
+
+
+@pytest.mark.models
+def test_tokenize():
+    if not os.environ.get("XAI_API_KEY"):
+        pytest.skip("No xAI API key found; skipping test")
+    tokenizer = GrokAPITokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="xai/grok-3-beta", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in ["otter", "", "", ""]]
+
+
+@pytest.mark.models
+def test_encode():
+    if not os.environ.get("XAI_API_KEY"):
+        pytest.skip("No xAI API key found; skipping test")
+    tokenizer = GrokAPITokenizer(cache_config=BlackHoleCacheConfig())
+    request = TokenizationRequest(tokenizer="xai/grok-3-beta", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [142507, 11637, 294, 294]]

helm/{proxy/tokenizers → tokenizers}/test_huggingface_tokenizer.py

@@ -10,14 +10,18 @@ from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from .huggingface_tokenizer import HuggingFaceTokenizer
+from helm.tokenizers.huggingface_tokenizer import HuggingFaceTokenizer
 
 
 class TestHuggingFaceGPT2Tokenizer:
     def setup_method(self, method):
         cache_file = tempfile.NamedTemporaryFile(delete=False)
         self.cache_path: str = cache_file.name
-        self.tokenizer = HuggingFaceTokenizer(SqliteCacheConfig(self.cache_path))
+        self.tokenizer = HuggingFaceTokenizer(
+            SqliteCacheConfig(self.cache_path),
+            tokenizer_name="huggingface/gpt2",
+            pretrained_model_name_or_path="openai-community/gpt2",
+        )
 
     def teardown_method(self, method):
         os.remove(self.cache_path)
@@ -124,6 +128,9 @@ class TestHuggingFaceTokenizer:
     def test_get_santacoder(self):
         TestHuggingFaceTokenizer.verify_get_tokenizer("bigcode/santacoder", 62)
 
+    def test_get_clip_tokenizer(self):
+        TestHuggingFaceTokenizer.verify_get_tokenizer("openai/clip-vit-large-patch14", 50)
+
     def test_gpt2_tokenize_eos(self):
         eos_token: str = "<|endoftext|>"
         wrapped_tokenizer = HuggingFaceTokenizer.get_tokenizer("huggingface/gpt2", pretrained_model_name_or_path="gpt2")

helm/tokenizers/test_simple_tokenizer.py

@@ -0,0 +1,33 @@
+from helm.common.tokenization_request import (
+    DecodeRequest,
+    TokenizationRequest,
+    TokenizationToken,
+)
+from helm.tokenizers.simple_tokenizer import SimpleTokenizer
+
+
+def test_simple_tokenizer_tokenize():
+    tokenizer = SimpleTokenizer()
+    request = TokenizationRequest(tokenizer="simple/tokenizer1", text="otter 🦦")
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in ["o", "t", "t", "e", "r", " ", "🦦"]]
+
+
+def test_simple_tokenizer_encode():
+    tokenizer = SimpleTokenizer()
+    request = TokenizationRequest(tokenizer="simple/tokenizer1", text="otter 🦦", encode=True)
+    result = tokenizer.tokenize(request)
+    assert result.success
+    assert not result.cached
+    assert result.tokens == [TokenizationToken(token) for token in [111, 116, 116, 101, 114, 32, 129446]]
+
+
+def test_simple_tokenizer_decode():
+    tokenizer = SimpleTokenizer()
+    request = DecodeRequest(tokenizer="simple/tokenizer1", tokens=[111, 116, 116, 101, 114, 32, 129446])
+    result = tokenizer.decode(request)
+    assert result.success
+    assert not result.cached
+    assert result.text == "otter 🦦"

helm/{proxy/tokenizers → tokenizers}/test_yalm_tokenizer.py

@@ -10,7 +10,7 @@ from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationRequestResult,
 )
-from .yalm_tokenizer import YaLMTokenizer
+from helm.tokenizers.yalm_tokenizer import YaLMTokenizer
 
 
 class TestYaLMTokenizer:

helm/{proxy/tokenizers → tokenizers}/tiktoken_tokenizer.py

@@ -1,7 +1,7 @@
 from typing import Any, Dict
 
 from helm.common.optional_dependencies import handle_module_not_found_error
-from .caching_tokenizer import CachingTokenizer
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
 
 try:
     import tiktoken

helm/{proxy/tokenizers → tokenizers}/tokenizer.py

@@ -41,7 +41,9 @@ def cleanup_str(token: str, tokenizer_name: Optional[str] = None) -> str:
         "together",
     ]:
         return token.replace("▁", " ")
-    elif tokenizer_name is not None and tokenizer_name.startswith("huggingface"):
+    elif tokenizer_name is not None and (
+        tokenizer_name.startswith("huggingface") or tokenizer_name == "anthropic/claude"
+    ):
         return token.replace("Ġ", " ")
     return token
 

helm/{proxy/tokenizers → tokenizers}/vertexai_tokenizer.py

@@ -9,7 +9,7 @@ from helm.common.tokenization_request import (
     TokenizationRequest,
     TokenizationToken,
 )
-from helm.proxy.tokenizers.caching_tokenizer import CachingTokenizer
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
 from helm.proxy.retry import NonRetriableException
 
 try:

helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer.py

@@ -1,9 +1,9 @@
 from typing import Any, Dict
 
 from helm.common.cache import CacheConfig
-from .caching_tokenizer import CachingTokenizer
-from .tokenizer import cleanup_tokens
-from .yalm_tokenizer_data.yalm_tokenizer import YaLMTokenizer as YaLMTokenizerInternal
+from helm.tokenizers.caching_tokenizer import CachingTokenizer
+from helm.tokenizers.tokenizer import cleanup_tokens
+from helm.tokenizers.yalm_tokenizer_data.yalm_tokenizer import YaLMTokenizer as YaLMTokenizerInternal
 
 
 class YaLMTokenizer(CachingTokenizer):
@@ -21,9 +21,11 @@ class YaLMTokenizer(CachingTokenizer):
         # This is a problem because then tokenize(" Hello", encode=False) == tokenize("Hello", encode=False)
         # That is why we manually replace "▁" with a space.
         return {
-            "tokens": token_ids
-            if request["encode"]
-            else cleanup_tokens(self._tokenizer.convert_ids_to_tokens(token_ids), request["tokenizer"])
+            "tokens": (
+                token_ids
+                if request["encode"]
+                else cleanup_tokens(self._tokenizer.convert_ids_to_tokens(token_ids), request["tokenizer"])
+            )
         }
 
     def _decode_do_it(self, request: Dict[str, Any]) -> Dict[str, Any]:

helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer_data/test_yalm_tokenizer.py

@@ -1,7 +1,7 @@
 from typing import List
 
 from helm.common.general import singleton
-from .yalm_tokenizer import YaLMTokenizer
+from helm.tokenizers.yalm_tokenizer_data.yalm_tokenizer import YaLMTokenizer
 
 
 class TestYaLMTokenizer:

helm/{proxy/tokenizers → tokenizers}/yalm_tokenizer_data/yalm_tokenizer.py

@@ -16,7 +16,7 @@ adapted from https://github.com/yandex/YaLM-100B/blob/main/megatron_lm/megatron/
 """
 
 
-YALM_TOKENIZER_PACKAGE: str = "helm.proxy.tokenizers.yalm_tokenizer_data"
+YALM_TOKENIZER_PACKAGE: str = "helm.tokenizers.yalm_tokenizer_data"
 YALM_TOKENIZER_VOCAB_FILENAME: str = "voc_100b.sp"
 
 

crfm_helm-0.4.0.dist-info/METADATA

@@ -1,264 +0,0 @@
-Metadata-Version: 2.1
-Name: crfm-helm
-Version: 0.4.0
-Summary: Benchmark for language models
-Home-page: https://github.com/stanford-crfm/helm
-Author: Stanford CRFM
-Author-email: contact-crfm@stanford.edu
-License: Apache License 2.0
-Keywords: language models benchmarking
-Classifier: Programming Language :: Python :: 3 :: Only
-Classifier: Programming Language :: Python :: 3.8
-Classifier: License :: OSI Approved :: Apache Software License
-Requires-Python: <3.11,>=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: cattrs ~=22.2.0
-Requires-Dist: dacite ~=1.6.0
-Requires-Dist: importlib-resources ~=5.10.0
-Requires-Dist: Mako ~=1.2.3
-Requires-Dist: numpy ~=1.23.3
-Requires-Dist: pyhocon ~=0.3.59
-Requires-Dist: retrying ~=1.3.4
-Requires-Dist: spacy ~=3.5.3
-Requires-Dist: tqdm ~=4.64.1
-Requires-Dist: zstandard ~=0.18.0
-Requires-Dist: sqlitedict ~=1.7.0
-Requires-Dist: bottle ~=0.12.23
-Requires-Dist: datasets ~=2.5.2
-Requires-Dist: pyarrow >=11.0.0
-Requires-Dist: nltk ~=3.7
-Requires-Dist: pyext ~=0.7
-Requires-Dist: rouge-score ~=0.1.2
-Requires-Dist: scipy ~=1.10.0
-Requires-Dist: uncertainty-calibration ~=0.1.4
-Requires-Dist: scikit-learn ~=1.1.2
-Requires-Dist: transformers ~=4.33.1
-Requires-Dist: torch <3.0.0,>=1.12.1
-Requires-Dist: torchvision <3.0.0,>=0.13.1
-Requires-Dist: google-api-python-client ~=2.64.0
-Provides-Extra: aleph-alpha
-Requires-Dist: aleph-alpha-client ~=2.14.0 ; extra == 'aleph-alpha'
-Requires-Dist: tokenizers ~=0.13.3 ; extra == 'aleph-alpha'
-Provides-Extra: all
-Requires-Dist: crfm-helm[proxy-server] ; extra == 'all'
-Requires-Dist: crfm-helm[human-evaluation] ; extra == 'all'
-Requires-Dist: crfm-helm[scenarios] ; extra == 'all'
-Requires-Dist: crfm-helm[metrics] ; extra == 'all'
-Requires-Dist: crfm-helm[plots] ; extra == 'all'
-Requires-Dist: crfm-helm[slurm] ; extra == 'all'
-Requires-Dist: crfm-helm[cleva] ; extra == 'all'
-Requires-Dist: crfm-helm[images] ; extra == 'all'
-Requires-Dist: crfm-helm[models] ; extra == 'all'
-Requires-Dist: crfm-helm[mongo] ; extra == 'all'
-Provides-Extra: anthropic
-Requires-Dist: anthropic ~=0.2.5 ; extra == 'anthropic'
-Requires-Dist: websocket-client ~=1.3.2 ; extra == 'anthropic'
-Provides-Extra: cleva
-Requires-Dist: unidecode ==1.3.6 ; extra == 'cleva'
-Requires-Dist: pypinyin ==0.49.0 ; extra == 'cleva'
-Requires-Dist: jieba ==0.42.1 ; extra == 'cleva'
-Requires-Dist: opencc ==1.1.6 ; extra == 'cleva'
-Requires-Dist: langdetect ==1.0.9 ; extra == 'cleva'
-Provides-Extra: dev
-Requires-Dist: pytest ~=7.2.0 ; extra == 'dev'
-Requires-Dist: black ~=22.10.0 ; extra == 'dev'
-Requires-Dist: mypy ~=1.5.1 ; extra == 'dev'
-Requires-Dist: pre-commit ~=2.20.0 ; extra == 'dev'
-Requires-Dist: flake8 ~=5.0.4 ; extra == 'dev'
-Provides-Extra: google
-Requires-Dist: google-cloud-aiplatform ~=1.36.4 ; extra == 'google'
-Provides-Extra: human-evaluation
-Requires-Dist: scaleapi ~=2.13.0 ; extra == 'human-evaluation'
-Requires-Dist: surge-api ~=1.1.0 ; extra == 'human-evaluation'
-Provides-Extra: images
-Requires-Dist: accelerate ~=0.23.0 ; extra == 'images'
-Requires-Dist: pillow ~=9.4.0 ; extra == 'images'
-Provides-Extra: metrics
-Requires-Dist: numba ~=0.56.4 ; extra == 'metrics'
-Requires-Dist: pytrec-eval ==0.5 ; extra == 'metrics'
-Requires-Dist: sacrebleu ~=2.2.1 ; extra == 'metrics'
-Requires-Dist: summ-eval ~=0.892 ; extra == 'metrics'
-Provides-Extra: models
-Requires-Dist: crfm-helm[aleph-alpha] ; extra == 'models'
-Requires-Dist: crfm-helm[anthropic] ; extra == 'models'
-Requires-Dist: crfm-helm[google] ; extra == 'models'
-Requires-Dist: crfm-helm[openai] ; extra == 'models'
-Requires-Dist: crfm-helm[tsinghua] ; extra == 'models'
-Requires-Dist: crfm-helm[yandex] ; extra == 'models'
-Provides-Extra: mongo
-Requires-Dist: pymongo ~=4.2.0 ; extra == 'mongo'
-Provides-Extra: openai
-Requires-Dist: openai ~=0.27.8 ; extra == 'openai'
-Requires-Dist: tiktoken ~=0.3.3 ; extra == 'openai'
-Provides-Extra: plots
-Requires-Dist: colorcet ~=3.0.1 ; extra == 'plots'
-Requires-Dist: matplotlib ~=3.6.0 ; extra == 'plots'
-Requires-Dist: seaborn ~=0.11.0 ; extra == 'plots'
-Provides-Extra: proxy-server
-Requires-Dist: gunicorn ~=20.1.0 ; extra == 'proxy-server'
-Provides-Extra: scenarios
-Requires-Dist: gdown ~=4.4.0 ; extra == 'scenarios'
-Requires-Dist: sympy ~=1.11.1 ; extra == 'scenarios'
-Requires-Dist: xlrd ~=2.0.1 ; extra == 'scenarios'
-Provides-Extra: slurm
-Requires-Dist: simple-slurm ~=0.2.6 ; extra == 'slurm'
-Provides-Extra: tsinghua
-Requires-Dist: icetk ~=0.0.4 ; extra == 'tsinghua'
-Provides-Extra: yandex
-Requires-Dist: sentencepiece ~=0.1.97 ; extra == 'yandex'
-
-<!--intro-start-->
-
-# Holistic Evaluation of Language Models
-
-[comment]: <> (When using the img tag, which allows us to specify size, src has to be a URL.)
-<img src="https://github.com/stanford-crfm/helm/raw/main/src/helm/benchmark/static/images/helm-logo.png" alt=""  width="800"/>
-
-Welcome! The **`crfm-helm`** Python package contains code used in the **Holistic Evaluation of Language Models** project ([paper](https://arxiv.org/abs/2211.09110), [website](https://crfm.stanford.edu/helm/latest/)) by [Stanford CRFM](https://crfm.stanford.edu/). This package includes the following features:
-
-- Collection of datasets in a standard format (e.g., NaturalQuestions)
-- Collection of models accessible via a unified API (e.g., GPT-3, MT-NLG, OPT, BLOOM)
-- Collection of metrics beyond accuracy (efficiency, bias, toxicity, etc.)
-- Collection of perturbations for evaluating robustness and fairness (e.g., typos, dialect)
-- Modular framework for constructing prompts from datasets
-- Proxy server for managing accounts and providing unified interface to access models
-<!--intro-end-->
-
-To get started, refer to [the documentation on Read the Docs](https://crfm-helm.readthedocs.io/) for how to install and run the package.
-
-## Directory Structure
-
-The directory structure for this repo is as follows
-
-```
-├── docs # MD used to generate readthedocs
-│
-├── scripts # Python utility scripts for HELM
-│ ├── cache
-│ ├── data_overlap # Calculate train test overlap
-│ │ ├── common
-│ │ ├── scenarios
-│ │ └── test
-│ ├── efficiency
-│ ├── fact_completion
-│ ├── offline_eval
-│ └── scale
-└── src
-├── helm # Benchmarking Scripts for HELM
-│ │
-│ ├── benchmark # Main Python code for running HELM
-│ │ │
-│ │ └── static # Current JS (Jquery) code for rendering front-end
-│ │ │
-│ │ └── ...
-│ │
-│ ├── common # Additional Python code for running HELM
-│ │
-│ └── proxy # Python code for external web requests
-│
-└── helm-frontend # New React Front-end
-```
-
-# Tutorial
-
-This tutorial will explain how to use the HELM command line tools to run benchmarks, aggregate statistics, and visualize results.
-
-We will run two runs using the `mmlu` scenario on the `openai/gpt2` model. The `mmlu` scenario implements the **Massive Multitask Language (MMLU)** benchmark from [this paper](https://arxiv.org/pdf/2009.03300.pdf), and consists of a Question Answering (QA) task using a dataset with questions from 57 subjects such as elementary mathematics, US history, computer science, law, and more. Note that GPT-2 performs poorly on MMLU, so this is just a proof of concept. We will run two runs: the first using questions about anatomy, and the second using questions about philosophy.
-
-## Using `helm-run`
-
-`helm-run` is a command line tool for running benchmarks.
-
-To run this benchmark using the HELM command-line tools, we need to specify **run spec descriptions** that describes the desired runs. For this example, the run spec descriptions are `mmlu:subject=anatomy,model=openai/gpt2` (for anatomy) and `mmlu:subject=philosophy,model=openai/gpt2` (for philosophy).
-
-Next, we need to create a **run spec configuration file** contining these run spec descriptions. A run spec configuration file is a text file containing `RunEntries` serialized to JSON, where each entry in `RunEntries` contains a run spec description. The `description` field of each entry should be a **run spec description**. Create a text file named `run_specs.conf` with the following contents:
-
-```
-entries: [
-  {description: "mmlu:subject=anatomy,model=openai/gpt2", priority: 1},
-  {description: "mmlu:subject=philosophy,model=openai/gpt2", priority: 1},
-]
-```
-
-We will now use `helm-run` to execute the runs that have been specified in this run spec configuration file. Run this command:
-
-```
-helm-run --conf-paths run_specs.conf --suite v1 --max-eval-instances 10
-```
-
-The meaning of the additional arguments are as follows:
-
-- `--suite` specifies a subdirectory under the output directory in which all the output will be placed.
-- `--max-eval-instances` limits evaluation to only the first *N* inputs (i.e. instances) from the benchmark.
-
-`helm-run` creates an environment directory environment and an output directory by default.
-
--  The environment directory is `prod_env/` by default and can be set using `--local-path`. Credentials for making API calls should be added to a `credentials.conf` file in this directory.
--  The output directory is `benchmark_output/` by default and can be set using `--output-path`.
-
-After running this command, navigate to the `benchmark_output/runs/v1/` directory. This should contain a two sub-directories named `mmlu:subject=anatomy,model=openai_gpt2` and `mmlu:subject=philosophy,model=openai_gpt2`. Note that the names of these sub-directories is based on the run spec descriptions we used earlier, but with `/` replaced with `_`.
-
-Each output sub-directory will contain several JSON files that were generated during the corresponding run:
-
-- `run_spec.json` contains the `RunSpec`, which specifies the scenario, adapter and metrics for the run.
-- `scenario.json` contains a serialized `Scenario`, which contains the scenario for the run and specifies the instances (i.e. inputs) used.
-- `scenario_state.json` contains a serialized `ScenarioState`, which contains every request to and response from the model.
-- `per_instance_stats.json` contains a serialized list of `PerInstanceStats`, which contains the statistics produced for the metrics for each instance (i.e. input).
-- `stats.json` contains a serialized list of `PerInstanceStats`, which contains the statistics produced for the metrics, aggregated across all instances (i.e. inputs).
-
-`helm-run` provides additional arguments that can be used to filter out `--models-to-run`, `--groups-to-run` and `--priority`. It can be convenient to create a large `run_specs.conf` file containing every run spec description of interest, and then use these flags to filter down the RunSpecs to actually run. As an example, the main `run_specs.conf` file used for the HELM benchmarking paper can be found [here](https://github.com/stanford-crfm/helm/blob/main/src/helm/benchmark/presentation/run_specs.conf).
-
-## Using `helm-summarize`
-
-The `helm-summarize` reads the output files of `helm-run` and computes aggregate statistics across runs. Run the following:
-
-```
-helm-summarize --suite v1
-```
-
-This reads the pre-existing files in `benchmark_output/runs/v1/` that were written by `helm-run` previously, and writes the following new files back to `benchmark_output/runs/v1/`:
-
-- `summary.json` contains a serialized `ExecutiveSummary` with a date and suite name.
-- `run_specs.json` contains the run spec descriptions for all the runs.
-- `runs.json` contains serialized list of `Run`, which contains the run path, run spec and adapter spec and statistics for each run.
-- `groups.json` contains a serialized list of `Table`, each containing information about groups in a group category.
-- `groups_metadata.json` contains a list of all the groups along with a human-readable description and a taxonomy.
-
-Additionally, for each group and group-relavent metric, it will output a pair of files: `benchmark_output/runs/v1/groups/latex/<group_name>_<metric_name>.tex` and `benchmark_output/runs/v1/groups/json/<group_name>_<metric_name>.json`. These files contain the statistics for that metric from each run within the group.
-
-<!--
-# TODO(#1441): Enable plots
-
-## Using `helm-create-plots`
-
-The `helm-create-plots` reads the `groups` directory created by `helm-summarize` and creates plots, equivalent to those use in the HELM paper. Run the following:
-
-```
-helm-create-plots --suite v1
-```
-
-This reads the pre-existing files in `benchmark_output/runs/v1/groups` that were written by `helm-summarize` previously,
-and creates plots (`.png` or `.pdf`) at `benchmark_output/runs/v1/plots`.
-
--->
-
-## Using `helm-server`
-
-Finally, the `helm-server` command launches a web server to visualize the output files of `helm-run` and `helm-benchmark`. Run:
-
-```
-helm-server
-```
-
-Open a browser and go to http://localhost:8000/ to view the visualization. You should see a similar view as [live website for the paper](https://crfm.stanford.edu/helm/v1.0/), but for the data from your benchmark runs. The website has three main sections:
-
-- **Models** contains a list of available models.
-- **Scenarios** contains a list of available scenarios.
-- **Results** contains results from the runs, organized into groups and categories of groups.
-- **Raw Runs** contains a searchable list of runs.
-
-## Other Tips
-
-- The suite name can be used as a versioning mechanism to separate runs using different versions of scenarios or models.
-- Tools such as [`jq`](https://stedolan.github.io/jq/) are useful for examining the JSON output files on the command line.