crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/benchmark/test_model_properties.py

@@ -1,1570 +0,0 @@
-"""Temporary test for preserving invariants during the model / tokenizer / window service refactor.
-
-Delete this after the refactor is done."""
-
-import pytest
-from tempfile import TemporaryDirectory
-from typing import Any
-from helm.benchmark.config_registry import register_builtin_configs_from_helm_package
-from helm.benchmark.model_deployment_registry import (
-    ClientSpec,
-    ModelDeployment,
-    WindowServiceSpec,
-    get_model_deployment,
-)
-from helm.benchmark.model_metadata_registry import get_model_metadata, ModelMetadata
-from helm.benchmark.tokenizer_config_registry import TokenizerConfig, TokenizerSpec
-from helm.benchmark.window_services.test_utils import get_tokenizer_service
-
-from helm.benchmark.window_services.window_service_factory import WindowServiceFactory
-from helm.proxy.clients.auto_client import AutoClient
-from helm.proxy.tokenizers.auto_tokenizer import AutoTokenizer
-from collections import defaultdict
-
-
-_BUILT_IN_TOKENIZER_CONFIGS = [
-    TokenizerConfig(
-        name="simple/model1",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.simple_tokenizer.SimpleTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="<s>",
-    ),
-    TokenizerConfig(
-        name="neurips/local",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.http_model_tokenizer.HTTPModelTokenizer"),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="<|endoftext|>",
-    ),
-    TokenizerConfig(
-        name="ai21/j1",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.ai21_tokenizer.AI21Tokenizer"),
-        end_of_text_token=" ",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="AlephAlpha/luminous-base",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"),
-        end_of_text_token="",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="AlephAlpha/luminous-extended",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"),
-        end_of_text_token="",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="AlephAlpha/luminous-supreme",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.aleph_alpha_tokenizer.AlephAlphaTokenizer"),
-        end_of_text_token="",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="huggingface/gpt2",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="<|endoftext|>",
-    ),
-    TokenizerConfig(
-        name="microsoft/gpt2",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="<<",
-    ),
-    TokenizerConfig(
-        name="writer/gpt2",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="anthropic/claude",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.anthropic_tokenizer.AnthropicTokenizer"),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="<|endoftext|>",
-    ),
-    TokenizerConfig(
-        name="bigscience/bloom",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="</s>",
-    ),
-    TokenizerConfig(
-        name="bigscience/T0pp",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="cohere/cohere",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.cohere_tokenizer.CohereTokenizer"),
-        end_of_text_token="",
-        prefix_token=":",
-    ),
-    TokenizerConfig(
-        name="EleutherAI/gpt-j-6B",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="<|endoftext|>",
-    ),
-    TokenizerConfig(
-        name="EleutherAI/gpt-neox-20b",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="<|endoftext|>",
-    ),
-    TokenizerConfig(
-        name="hf-internal-testing/llama-tokenizer",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="<s>",
-    ),
-    TokenizerConfig(
-        name="meta-llama/Llama-2-7b-hf",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="<s>",
-    ),
-    TokenizerConfig(
-        name="mistralai/Mistral-7B-v0.1",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="<s>",
-    ),
-    TokenizerConfig(
-        name="tiiuae/falcon-7b",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="bigcode/santacoder",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="<|endoftext|>",
-    ),
-    TokenizerConfig(
-        name="bigcode/starcoder",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="<|endoftext|>",
-    ),
-    TokenizerConfig(
-        name="google/t5-11b",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="google/flan-t5-xxl",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="google/ul2",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="google/mt5-base",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="facebook/opt-66b",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="</s>",
-    ),
-    TokenizerConfig(
-        name="openai/cl100k_base",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.tiktoken_tokenizer.TiktokenTokenizer"),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="<|endoftext|>",
-    ),
-    TokenizerConfig(
-        name="TsinghuaKEG/ice",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.ice_tokenizer.ICETokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="",
-    ),
-    TokenizerConfig(
-        name="Yandex/yalm",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.yalm_tokenizer.YaLMTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="</s>",
-    ),
-    TokenizerConfig(
-        name="lightningai/lit-gpt",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.lit_gpt_tokenizer.LitGPTTokenizer", args={}),
-        end_of_text_token="<|endoftext|>",
-        prefix_token="<|endoftext|>",
-    ),
-    TokenizerConfig(
-        name="HuggingFaceM4/idefics-9b",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="<s>",
-    ),
-    TokenizerConfig(
-        name="HuggingFaceM4/idefics-9b-instruct",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="<s>",
-    ),
-    TokenizerConfig(
-        name="HuggingFaceM4/idefics-80b",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="<s>",
-    ),
-    TokenizerConfig(
-        name="HuggingFaceM4/idefics-80b-instruct",
-        tokenizer_spec=TokenizerSpec(class_name="helm.proxy.tokenizers.huggingface_tokenizer.HuggingFaceTokenizer"),
-        end_of_text_token="</s>",
-        prefix_token="<s>",
-    ),
-]
-
-
-_BUILT_IN_MODEL_DEPLOYMENTS = [
-    ModelDeployment(
-        name="neurips/local",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.http_model_client.HTTPModelClient"),
-        tokenizer_name="neurips/local",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="ai21/j1-jumbo",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
-        tokenizer_name="ai21/j1",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="ai21/j1-grande",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
-        tokenizer_name="ai21/j1",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="ai21/j1-grande-v2-beta",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
-        tokenizer_name="ai21/j1",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="ai21/j1-large",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
-        tokenizer_name="ai21/j1",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="ai21/j2-jumbo",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
-        tokenizer_name="ai21/j1",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.wider_ai21_window_service.AI21Jurassic2JumboWindowService",
-            args={},
-        ),
-        max_sequence_length=6000,
-    ),
-    ModelDeployment(
-        name="ai21/j2-grande",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
-        tokenizer_name="ai21/j1",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="ai21/j2-large",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.ai21_client.AI21Client"),
-        tokenizer_name="ai21/j1",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="AlephAlpha/luminous-base",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.aleph_alpha_client.AlephAlphaClient"),
-        tokenizer_name="AlephAlpha/luminous-base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="AlephAlpha/luminous-extended",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.aleph_alpha_client.AlephAlphaClient"),
-        tokenizer_name="AlephAlpha/luminous-extended",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="AlephAlpha/luminous-supreme",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.aleph_alpha_client.AlephAlphaClient"),
-        tokenizer_name="AlephAlpha/luminous-supreme",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="anthropic/stanford-online-all-v4-s3",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.anthropic_client.AnthropicLegacyClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=8192,
-    ),
-    ModelDeployment(
-        name="anthropic/claude-2.0",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.anthropic_client.AnthropicClient"),
-        tokenizer_name="anthropic/claude",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=8000,
-        max_sequence_and_generated_tokens_length=9016,
-    ),
-    ModelDeployment(
-        name="anthropic/claude-2.1",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.anthropic_client.AnthropicClient"),
-        tokenizer_name="anthropic/claude",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=8000,
-        max_sequence_and_generated_tokens_length=9016,
-    ),
-    ModelDeployment(
-        name="anthropic/claude-v1.3",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.anthropic_client.AnthropicClient"),
-        tokenizer_name="anthropic/claude",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=8000,
-        max_sequence_and_generated_tokens_length=9016,
-    ),
-    ModelDeployment(
-        name="anthropic/claude-instant-v1",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.anthropic_client.AnthropicClient"),
-        tokenizer_name="anthropic/claude",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=8000,
-        max_sequence_and_generated_tokens_length=9016,
-    ),
-    ModelDeployment(
-        name="together/bloom",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="bigscience/bloom",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/t0pp",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="bigscience/T0pp",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.t0pp_window_service.T0ppWindowService"
-        ),
-        max_sequence_length=1024,
-    ),
-    ModelDeployment(
-        name="cohere/xlarge-20220609",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
-        tokenizer_name="cohere/cohere",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-        ),
-        max_sequence_length=2047,
-        max_request_length=2048,
-    ),
-    ModelDeployment(
-        name="cohere/xlarge-20221108",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
-        tokenizer_name="cohere/cohere",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-        ),
-        max_sequence_length=2047,
-        max_request_length=2048,
-    ),
-    ModelDeployment(
-        name="cohere/large-20220720",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
-        tokenizer_name="cohere/cohere",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-        ),
-        max_sequence_length=2047,
-        max_request_length=2048,
-    ),
-    ModelDeployment(
-        name="cohere/medium-20220720",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
-        tokenizer_name="cohere/cohere",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-        ),
-        max_sequence_length=2047,
-        max_request_length=2048,
-    ),
-    ModelDeployment(
-        name="cohere/medium-20221108",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
-        tokenizer_name="cohere/cohere",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-        ),
-        max_sequence_length=2047,
-        max_request_length=2048,
-    ),
-    ModelDeployment(
-        name="cohere/small-20220720",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
-        tokenizer_name="cohere/cohere",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-        ),
-        max_sequence_length=2047,
-        max_request_length=2048,
-    ),
-    ModelDeployment(
-        name="cohere/command-medium-beta",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
-        tokenizer_name="cohere/cohere",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
-        ),
-        max_sequence_length=2019,
-        max_request_length=2020,
-    ),
-    ModelDeployment(
-        name="cohere/command-xlarge-beta",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
-        tokenizer_name="cohere/cohere",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
-        ),
-        max_sequence_length=2019,
-        max_request_length=2020,
-    ),
-    ModelDeployment(
-        name="cohere/command",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
-        tokenizer_name="cohere/cohere",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
-        ),
-        max_sequence_length=2019,
-        max_request_length=2020,
-    ),
-    ModelDeployment(
-        name="cohere/command-light",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.cohere_client.CohereClient"),
-        tokenizer_name="cohere/cohere",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
-        ),
-        max_sequence_length=2019,
-        max_request_length=2020,
-    ),
-    ModelDeployment(
-        name="together/gpt-j-6b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-j-6B",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/gpt-neox-20b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/pythia-1b-v0",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/pythia-2.8b-v0",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/pythia-6.9b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/pythia-12b-v0",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/llama-7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="hf-internal-testing/llama-tokenizer",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="together/llama-13b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="hf-internal-testing/llama-tokenizer",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="together/llama-30b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="hf-internal-testing/llama-tokenizer",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="together/llama-65b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="hf-internal-testing/llama-tokenizer",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="together/llama-2-7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="meta-llama/Llama-2-7b-hf",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4094,
-    ),
-    ModelDeployment(
-        name="together/llama-2-13b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="meta-llama/Llama-2-7b-hf",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4094,
-    ),
-    ModelDeployment(
-        name="together/llama-2-70b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="meta-llama/Llama-2-7b-hf",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4094,
-    ),
-    ModelDeployment(
-        name="together/alpaca-7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="hf-internal-testing/llama-tokenizer",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="together/vicuna-7b-v1.3",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="hf-internal-testing/llama-tokenizer",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="together/vicuna-13b-v1.3",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="hf-internal-testing/llama-tokenizer",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="together/mistral-7b-v0.1",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="mistralai/Mistral-7B-v0.1",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4095,
-    ),
-    ModelDeployment(
-        name="together/mpt-7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/mpt-instruct-7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/mpt-30b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/mpt-instruct-30b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/falcon-7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="tiiuae/falcon-7b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="together/falcon-7b-instruct",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="tiiuae/falcon-7b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="together/falcon-40b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="tiiuae/falcon-7b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="together/falcon-40b-instruct",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="tiiuae/falcon-7b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2047,
-    ),
-    ModelDeployment(
-        name="gooseai/gpt-neo-20b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.goose_ai_client.GooseAIClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="gooseai/gpt-j-6b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.goose_ai_client.GooseAIClient"),
-        tokenizer_name="EleutherAI/gpt-j-6B",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="huggingface/gpt2",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.huggingface_client.HuggingFaceClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=1024,
-        max_request_length=1025,
-    ),
-    ModelDeployment(
-        name="huggingface/gpt-j-6b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.huggingface_client.HuggingFaceClient"),
-        tokenizer_name="EleutherAI/gpt-j-6B",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="huggingface/santacoder",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.huggingface_client.HuggingFaceClient"),
-        tokenizer_name="bigcode/santacoder",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="huggingface/starcoder",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.huggingface_client.HuggingFaceClient"),
-        tokenizer_name="bigcode/starcoder",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=8192,
-    ),
-    ModelDeployment(
-        name="together/t5-11b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="google/t5-11b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.t511b_window_service.T511bWindowService"
-        ),
-        max_sequence_length=511,
-    ),
-    ModelDeployment(
-        name="together/flan-t5-xxl",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="google/flan-t5-xxl",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.flan_t5_window_service.FlanT5WindowService"
-        ),
-        max_sequence_length=511,
-    ),
-    ModelDeployment(
-        name="together/ul2",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="google/ul2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.ul2_window_service.UL2WindowService"
-        ),
-        max_sequence_length=511,
-    ),
-    ModelDeployment(
-        name="google/text-bison@001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.vertexai_client.VertexAIClient"),
-        tokenizer_name="google/mt5-base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=6000,
-        max_sequence_and_generated_tokens_length=7000,
-    ),
-    ModelDeployment(
-        name="google/text-bison-32k",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.vertexai_client.VertexAIClient"),
-        tokenizer_name="google/mt5-base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=32000,
-        max_sequence_and_generated_tokens_length=32000,
-    ),
-    ModelDeployment(
-        name="google/text-unicorn@001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.vertexai_client.VertexAIClient"),
-        tokenizer_name="google/mt5-base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=6000,
-        max_sequence_and_generated_tokens_length=7000,
-    ),
-    ModelDeployment(
-        name="google/code-bison@001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.vertexai_client.VertexAIClient"),
-        tokenizer_name="google/mt5-base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=6000,
-        max_sequence_and_generated_tokens_length=7000,
-    ),
-    ModelDeployment(
-        name="google/code-bison-32k",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.vertexai_client.VertexAIClient"),
-        tokenizer_name="google/mt5-base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=32000,
-        max_sequence_and_generated_tokens_length=32000,
-    ),
-    ModelDeployment(
-        name="together/h3-2.7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=1024,
-        max_request_length=1025,
-    ),
-    ModelDeployment(
-        name="together/opt-175b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="facebook/opt-66b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/opt-66b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="facebook/opt-66b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/opt-6.7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="facebook/opt-66b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/opt-1.3b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="facebook/opt-66b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="microsoft/TNLGv2_530B",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.microsoft_client.MicrosoftClient"),
-        tokenizer_name="microsoft/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2047,
-        max_request_length=2048,
-    ),
-    ModelDeployment(
-        name="microsoft/TNLGv2_7B",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.microsoft_client.MicrosoftClient"),
-        tokenizer_name="microsoft/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2047,
-        max_request_length=2048,
-    ),
-    ModelDeployment(
-        name="openai/davinci",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/curie",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/babbage",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/ada",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/text-davinci-003",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4000,
-        max_request_length=4001,
-    ),
-    ModelDeployment(
-        name="openai/text-davinci-002",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4000,
-        max_request_length=4001,
-    ),
-    ModelDeployment(
-        name="openai/text-davinci-001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/text-curie-001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/text-babbage-001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/text-ada-001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/code-davinci-002",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4000,
-        max_request_length=4001,
-    ),
-    ModelDeployment(
-        name="openai/code-davinci-001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/code-cushman-001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/gpt-4-1106-preview",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="openai/cl100k_base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=128000,
-        max_request_length=128001,
-    ),
-    ModelDeployment(
-        name="openai/gpt-4-0314",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="openai/cl100k_base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=8192,
-        max_request_length=8193,
-    ),
-    ModelDeployment(
-        name="openai/gpt-4-32k-0314",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="openai/cl100k_base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=32768,
-        max_request_length=32769,
-    ),
-    ModelDeployment(
-        name="openai/gpt-4-0613",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="openai/cl100k_base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=8192,
-        max_request_length=8193,
-    ),
-    ModelDeployment(
-        name="openai/gpt-4-32k-0613",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="openai/cl100k_base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=32768,
-        max_request_length=32769,
-    ),
-    ModelDeployment(
-        name="openai/gpt-3.5-turbo-0301",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="openai/cl100k_base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4000,
-        max_request_length=4001,
-    ),
-    ModelDeployment(
-        name="openai/gpt-3.5-turbo-0613",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="openai/cl100k_base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4000,
-        max_request_length=4001,
-    ),
-    ModelDeployment(
-        name="openai/gpt-3.5-turbo-16k-0613",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="openai/cl100k_base",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=16000,
-        max_request_length=16001,
-    ),
-    ModelDeployment(
-        name="openai/text-similarity-davinci-001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/text-similarity-curie-001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/text-similarity-babbage-001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/text-similarity-ada-001",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="openai/text-embedding-ada-002",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.openai_client.OpenAIClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/gpt-jt-6b-v1",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-j-6B",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/gpt-neoxt-chat-base-20b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/redpajama-incite-base-3b-v1",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/redpajama-incite-instruct-3b-v1",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/redpajama-incite-base-7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/redpajama-incite-instruct-7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/glm",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="TsinghuaKEG/ice",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.ice_window_service.ICEWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="writer/palmyra-base",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
-        tokenizer_name="writer/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_sequence_and_generated_tokens_length=2048,
-    ),
-    ModelDeployment(
-        name="writer/palmyra-large",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
-        tokenizer_name="writer/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_sequence_and_generated_tokens_length=2048,
-    ),
-    ModelDeployment(
-        name="writer/palmyra-instruct-30",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
-        tokenizer_name="writer/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_sequence_and_generated_tokens_length=2048,
-    ),
-    ModelDeployment(
-        name="writer/palmyra-e",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
-        tokenizer_name="writer/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_sequence_and_generated_tokens_length=2048,
-    ),
-    ModelDeployment(
-        name="writer/silk-road",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
-        tokenizer_name="writer/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=8192,
-        max_sequence_and_generated_tokens_length=8192,
-    ),
-    ModelDeployment(
-        name="writer/palmyra-x",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.palmyra_client.PalmyraClient"),
-        tokenizer_name="writer/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=8192,
-        max_sequence_and_generated_tokens_length=8192,
-    ),
-    ModelDeployment(
-        name="together/yalm",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="Yandex/yalm",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.yalm_window_service.YaLMWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="nvidia/megatron-gpt2",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.megatron_client.MegatronClient"),
-        tokenizer_name="huggingface/gpt2",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=1024,
-    ),
-    ModelDeployment(
-        name="together/dolly-v2-3b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/dolly-v2-7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/dolly-v2-12b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-        max_request_length=2049,
-    ),
-    ModelDeployment(
-        name="together/stablelm-base-alpha-3b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4096,
-        max_request_length=4097,
-    ),
-    ModelDeployment(
-        name="together/stablelm-base-alpha-7b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.together_client.TogetherClient"),
-        tokenizer_name="EleutherAI/gpt-neox-20b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=4096,
-        max_request_length=4097,
-    ),
-    ModelDeployment(
-        name="lightningai/lit-gpt",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.lit_gpt_client.LitGPTClient", args={}),
-        model_name=None,
-        tokenizer_name="lightningai/lit-gpt",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService", args={}
-        ),
-        max_sequence_length=2048,
-        max_request_length=None,
-        max_sequence_and_generated_tokens_length=None,
-    ),
-    ModelDeployment(
-        name="HuggingFaceM4/idefics-9b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"),
-        tokenizer_name="HuggingFaceM4/idefics-9b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="HuggingFaceM4/idefics-9b-instruct",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"),
-        tokenizer_name="HuggingFaceM4/idefics-9b-instruct",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="HuggingFaceM4/idefics-80b",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"),
-        tokenizer_name="HuggingFaceM4/idefics-80b",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="HuggingFaceM4/idefics-80b-instruct",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"),
-        tokenizer_name="HuggingFaceM4/idefics-80b-instruct",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-    ModelDeployment(
-        name="simple/model1",
-        client_spec=ClientSpec(class_name="helm.proxy.clients.simple_client.SimpleClient"),
-        tokenizer_name="simple/model1",
-        window_service_spec=WindowServiceSpec(
-            class_name="helm.benchmark.window_services.default_window_service.DefaultWindowService"
-        ),
-        max_sequence_length=2048,
-    ),
-]
-
-
-_INT_MAX: int = 2**31 - 1
-
-
-def _full_class_name(obj: Any) -> str:
-    return f"{obj.__class__.__module__}.{obj.__class__.__name__}"
-
-
-# HACK: This looks like it should be done in a setup_class()
-# for the test below but apparently pytest first check the parametrize
-# before running the setup_class().
-# Therefore ALL_MODEL_DEPLOYMENTS is empty and no test would be run,
-# so we need to do this here.
-register_builtin_configs_from_helm_package()
-
-
-class TestModelProperties:
-    @pytest.mark.parametrize("deployment_name", [deployment.name for deployment in _BUILT_IN_MODEL_DEPLOYMENTS])
-    def test_models_has_window_service(self, deployment_name: str):
-        auto_client = AutoClient(defaultdict(str), "", "")
-        auto_tokenizer = AutoTokenizer(defaultdict(str), "", "")
-        model_deployments = {
-            model_deployment.name: model_deployment for model_deployment in _BUILT_IN_MODEL_DEPLOYMENTS
-        }
-        tokenizer_configs = {
-            tokenizer_config.name: tokenizer_config for tokenizer_config in _BUILT_IN_TOKENIZER_CONFIGS
-        }
-        with TemporaryDirectory() as tmpdir:
-            tokenizer_service = get_tokenizer_service(tmpdir)
-            deployment: ModelDeployment = get_model_deployment(deployment_name)
-            model: ModelMetadata = get_model_metadata(deployment.model_name or deployment_name)
-            # Can't test lit-gpt client because it requires manual dependencies
-            if "lit-gpt" in model.name:
-                return
-
-            # Can't test Llama 2 because it requires Hugging Face credentials
-            if "llama-2-" in model.name:
-                return
-
-            # Can't test Vertex AI because it requires Google credentials
-            if "text-bison" in model.name or "text-unicorn" in model.name:
-                return
-
-            client = auto_client._get_client(deployment_name)
-            window_service = WindowServiceFactory.get_window_service(deployment_name, tokenizer_service)
-            tokenizer_name = window_service.tokenizer_name
-            tokenizer = auto_tokenizer._get_tokenizer(tokenizer_name)
-
-            client_class_name = _full_class_name(client)
-            tokenizer_class_name = _full_class_name(tokenizer)
-            window_service_class_name = _full_class_name(window_service)
-
-            prefix_token = window_service.prefix_token
-            end_of_text_token = window_service.end_of_text_token
-
-            max_sequence_length = window_service.max_sequence_length
-            max_request_length = (
-                window_service.max_request_length
-                if window_service.max_request_length != window_service.max_sequence_length
-                else None
-            )
-            max_sequence_and_generated_tokens_length = (
-                window_service.max_sequence_and_generated_tokens_length
-                if window_service.max_sequence_and_generated_tokens_length != _INT_MAX
-                else None
-            )
-
-            model_deployment = ModelDeployment(
-                name=deployment_name,
-                client_spec=ClientSpec(class_name=client_class_name),
-                tokenizer_name=tokenizer_name,
-                window_service_spec=WindowServiceSpec(class_name=window_service_class_name),
-                max_sequence_length=max_sequence_length,
-                max_request_length=max_request_length,
-                max_sequence_and_generated_tokens_length=max_sequence_and_generated_tokens_length,
-            )
-            tokenizer_config = TokenizerConfig(
-                name=tokenizer_name,
-                tokenizer_spec=TokenizerSpec(class_name=tokenizer_class_name),
-                end_of_text_token=end_of_text_token,
-                prefix_token=prefix_token,
-            )
-            # NOTE: To generate the _BUILT_IN_MODEL_DEPLOYMENT and _BUILT_IN_TOKENIZER_CONFIGS lists above,
-            # print tokenizer_config and model_deployment here.
-
-            # Cannot directly compare model_deployment and deployment because they will have different values
-            # for deprecated for example as it is not specified in this file.
-            # In stead compare all the fields defined here.
-            test_model_deployment: ModelDeployment = model_deployments[deployment_name]
-            assert model_deployment.name == test_model_deployment.name
-            assert model_deployment.client_spec.class_name == test_model_deployment.client_spec.class_name
-            assert model_deployment.tokenizer_name == test_model_deployment.tokenizer_name
-            if model_deployment.window_service_spec:
-                assert test_model_deployment.window_service_spec
-                assert (
-                    model_deployment.window_service_spec.class_name
-                    == test_model_deployment.window_service_spec.class_name
-                )
-            else:
-                assert not test_model_deployment.window_service_spec
-            assert model_deployment.max_sequence_length == test_model_deployment.max_sequence_length
-            assert model_deployment.max_request_length == test_model_deployment.max_request_length
-            assert (
-                model_deployment.max_sequence_and_generated_tokens_length
-                == test_model_deployment.max_sequence_and_generated_tokens_length
-            )
-            # DefaultWindowService overrides the huggingface/gpt2 tokenizer with different special tokens,
-            # so there are currently two tokenizers named huggingface/gpt2
-            # TODO: Give DefaultWindowService's tokenizer a different name e.g. writer/palmyra
-            if tokenizer_name != "huggingface/gpt2":
-                assert tokenizer_configs[tokenizer_name] == tokenizer_config