crfm-helm 0.3.0__py3-none-any.whl → 0.5.0__py3-none-any.whl

helm/config/model_deployments.yaml

@@ -0,0 +1,1942 @@
+# This file defines all the model deployments that are supported by the Helm API.
+# Some models have several deployments, each with different parameters.
+
+# If you want to add a new deployment, you can technically do it here but we recommend
+# you to do it in prod_env/model_deployments.yaml instead.
+
+# Follow the template of this file to add a new deployment. You can copy paste this to get started:
+#    # This file defines all the model deployments that you do not want to be public.
+#    model_deployments: [] # Leave empty to disable private model deployments
+
+model_deployments:
+  - name: simple/model1
+    model_name: simple/model1
+    tokenizer_name: simple/tokenizer1
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.simple_client.SimpleClient"
+
+  # Adobe
+  - name: adobe/giga-gan
+    model_name: adobe/giga-gan
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.adobe_vision_client.AdobeVisionClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  # AI21 Labs
+
+  # J1 models are Deprecated by AI21 Labs
+  # API returns: Detail: Jurassic J1 models are deprecated
+  - name: ai21/j1-jumbo
+    deprecated: true
+    model_name: ai21/j1-jumbo
+    tokenizer_name: ai21/j1
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.ai21_client.AI21Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+
+  - name: ai21/j1-large
+    deprecated: true
+    model_name: ai21/j1-large
+    tokenizer_name: ai21/j1
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.ai21_client.AI21Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+
+  - name: ai21/j1-grande
+    deprecated: true
+    model_name: ai21/j1-grande
+    tokenizer_name: ai21/j1
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.ai21_client.AI21Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+
+  - name: ai21/j1-grande-v2-beta
+    deprecated: true
+    model_name: ai21/j1-grande-v2-beta
+    tokenizer_name: ai21/j1
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.ai21_client.AI21Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+
+  - name: ai21/j2-jumbo
+    model_name: ai21/j2-jumbo
+    tokenizer_name: ai21/j1
+    max_sequence_length: 6000
+    client_spec:
+      class_name: "helm.clients.ai21_client.AI21Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+
+  - name: ai21/j2-large
+    model_name: ai21/j2-large
+    tokenizer_name: ai21/j1
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.ai21_client.AI21Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+
+  - name: ai21/j2-grande
+    model_name: ai21/j2-grande
+    tokenizer_name: ai21/j1
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.ai21_client.AI21Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
+
+  # Aleph Alpha
+  - name: AlephAlpha/luminous-base
+    model_name: AlephAlpha/luminous-base
+    tokenizer_name: AlephAlpha/luminous-base
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.aleph_alpha_client.AlephAlphaClient"
+
+  - name: AlephAlpha/luminous-extended
+    model_name: AlephAlpha/luminous-extended
+    tokenizer_name: AlephAlpha/luminous-extended
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.aleph_alpha_client.AlephAlphaClient"
+
+  - name: AlephAlpha/luminous-supreme
+    model_name: AlephAlpha/luminous-supreme
+    tokenizer_name: AlephAlpha/luminous-supreme
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.aleph_alpha_client.AlephAlphaClient"
+
+  # TODO: Add luminous-world once it is released
+
+  - name: AlephAlpha/m-vader
+    model_name: AlephAlpha/m-vader
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.aleph_alpha_image_generation_client.AlephAlphaImageGenerationClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+
+  # Amazon
+  - name: amazon/titan-text-lite-v1
+    model_name: amazon/titan-text-lite-v1
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 4000
+    client_spec:
+      class_name: "helm.clients.bedrock_client.BedrockTitanClient"
+
+  - name: amazon/titan-tg1-large
+    model_name: amazon/titan-tg1-large
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 8000
+    client_spec:
+      class_name: "helm.clients.bedrock_client.BedrockTitanClient"
+
+  - name: amazon/titan-text-express-v1
+    model_name: amazon/titan-text-express-v1
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 8000
+    client_spec:
+      class_name: "helm.clients.bedrock_client.BedrockTitanClient"
+
+
+  # Anthropic
+  - name: anthropic/claude-v1.3
+    model_name: anthropic/claude-v1.3
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 8000
+    max_sequence_and_generated_tokens_length: 9016
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicClient"
+
+  - name: anthropic/claude-instant-v1
+    model_name: anthropic/claude-instant-v1
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 8000
+    max_sequence_and_generated_tokens_length: 9016
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicClient"
+
+  - name: anthropic/claude-instant-1.2
+    model_name: anthropic/claude-instant-1.2
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 8000
+    max_sequence_and_generated_tokens_length: 9016
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicClient"
+
+  - name: anthropic/claude-2.0
+    model_name: anthropic/claude-2.0
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 8000
+    max_sequence_and_generated_tokens_length: 9016
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicClient"
+
+  - name: anthropic/claude-2.1
+    model_name: anthropic/claude-2.1
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 8000
+    max_sequence_and_generated_tokens_length: 9016
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicClient"
+
+  - name: anthropic/claude-3-sonnet-20240229
+    model_name: anthropic/claude-3-sonnet-20240229
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+
+  - name: anthropic/claude-3-haiku-20240307
+    model_name: anthropic/claude-3-haiku-20240307
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+
+  - name: anthropic/claude-3-opus-20240229
+    model_name: anthropic/claude-3-opus-20240229
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+
+  - name: anthropic/stanford-online-all-v4-s3
+    deprecated: true # Closed model, not accessible via API
+    model_name: anthropic/stanford-online-all-v4-s3
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicLegacyClient"
+
+  # Cohere
+  - name: cohere/xlarge-20220609
+    model_name: cohere/xlarge-20220609
+    tokenizer_name: cohere/cohere
+    max_sequence_length: 2047
+    max_request_length: 2048
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  - name: cohere/large-20220720
+    model_name: cohere/large-20220720
+    tokenizer_name: cohere/cohere
+    max_sequence_length: 2047
+    max_request_length: 2048
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  - name: cohere/medium-20220720
+    model_name: cohere/medium-20220720
+    tokenizer_name: cohere/cohere
+    max_sequence_length: 2047
+    max_request_length: 2048
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  - name: cohere/small-20220720
+    model_name: cohere/small-20220720
+    tokenizer_name: cohere/cohere
+    max_sequence_length: 2047
+    max_request_length: 2048
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  - name: cohere/xlarge-20221108
+    model_name: cohere/xlarge-20221108
+    tokenizer_name: cohere/cohere
+    max_sequence_length: 2047
+    max_request_length: 2048
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  - name: cohere/medium-20221108
+    model_name: cohere/medium-20221108
+    tokenizer_name: cohere/cohere
+    max_sequence_length: 2047
+    max_request_length: 2048
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  - name: cohere/command-medium-beta
+    model_name: cohere/command-medium-beta
+    tokenizer_name: cohere/cohere
+    max_sequence_length: 2019
+    max_request_length: 2020
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  - name: cohere/command-xlarge-beta
+    model_name: cohere/command-xlarge-beta
+    tokenizer_name: cohere/cohere
+    max_sequence_length: 2019
+    max_request_length: 2020
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  - name: cohere/command
+    model_name: cohere/command
+    tokenizer_name: cohere/cohere
+    max_sequence_length: 2019 # TODO: verify this
+    max_request_length: 2020 # TODO: verify this
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  - name: cohere/command-light
+    model_name: cohere/command-light
+    tokenizer_name: cohere/cohere
+    max_sequence_length: 2019 # TODO: verify this
+    max_request_length: 2020 # TODO: verify this
+    client_spec:
+      class_name: "helm.clients.cohere_client.CohereClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  # Craiyon
+
+  - name: craiyon/dalle-mini
+    model_name: craiyon/dalle-mini
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle_mini_client.DALLEMiniClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: craiyon/dalle-mega
+    model_name: craiyon/dalle-mega
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle_mini_client.DALLEMiniClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  # Databricks
+
+  - name: together/dbrx-instruct
+    model_name: databricks/dbrx-instruct
+    tokenizer_name: databricks/dbrx-instruct
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  # DeepFloyd
+
+  - name: DeepFloyd/IF-I-M-v1.0
+    model_name: DeepFloyd/IF-I-M-v1.0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.deep_floyd_client.DeepFloydClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: DeepFloyd/IF-I-L-v1.0
+    model_name: DeepFloyd/IF-I-L-v1.0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.deep_floyd_client.DeepFloydClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: DeepFloyd/IF-I-XL-v1.0
+    model_name: DeepFloyd/IF-I-XL-v1.0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.deep_floyd_client.DeepFloydClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  # Deepseek
+
+  - name: together/deepseek-llm-67b-chat
+    model_name: deepseek-ai/deepseek-llm-67b-chat
+    tokenizer_name: deepseek-ai/deepseek-llm-67b-chat
+    max_sequence_length: 4095
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  # Gooseai
+
+  # TODO: Migrate these models to use OpenAIClient
+
+  ## EleutherAI
+  # - name: gooseai/gpt-neo-20b
+  #   model_name: eleutherai/gpt-neox-20b
+  #   tokenizer_name: EleutherAI/gpt-neox-20b
+  #   max_sequence_length: 2048
+  #   max_request_length: 2049
+  #   client_spec:
+  #     class_name: "helm.clients.goose_ai_client.GooseAIClient"
+
+  # - name: gooseai/gpt-j-6b
+  #   model_name: eleutherai/gpt-j-6b
+  #   tokenizer_name: EleutherAI/gpt-j-6B
+  #   max_sequence_length: 2048
+  #   max_request_length: 2049
+  #   client_spec:
+  #     class_name: "helm.clients.goose_ai_client.GooseAIClient"
+
+  # Google
+  # See: https://cloud.google.com/vertex-ai/docs/generative-ai/learn/model-versioning
+
+  ## Gemini
+  # See: https://ai.google.dev/models/gemini#model_variations
+  - name: google/gemini-pro
+    model_name: google/gemini-pro
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 30720
+    max_sequence_and_generated_tokens_length: 32768 # Officially max_sequence_length + 2048
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-1.0-pro-001
+    model_name: google/gemini-1.0-pro-001
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 30720
+    max_sequence_and_generated_tokens_length: 32768 # Officially max_sequence_length + 2048
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-pro-vision
+    model_name: google/gemini-pro-vision
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 12288
+    max_sequence_and_generated_tokens_length: 16384 # Officially max_sequence_length + 4096, in practice max_output_tokens <= 2048 for vision models
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-1.0-pro-vision-001
+    model_name: google/gemini-1.0-pro-vision-001
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 12288
+    max_sequence_and_generated_tokens_length: 16384
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-1.5-pro-preview-0409
+    model_name: google/gemini-1.5-pro-preview-0409
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  ## Gemma
+  - name: together/gemma-2b
+    model_name: google/gemma-2b
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 7167
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/gemma-2b-it
+    model_name: google/gemma-2b-it
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 7167
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/gemma-7b
+    model_name: google/gemma-7b
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 7167
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/gemma-7b-it
+    model_name: google/gemma-7b-it
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 7167
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  ## PaLM 2
+  - name: google/text-bison@001
+    model_name: google/text-bison@001
+    tokenizer_name: google/text-bison@001
+    max_sequence_length: 6000 # Officially 8192
+    max_sequence_and_generated_tokens_length: 7000 # Officially 9216
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+  - name: google/text-bison@002
+    model_name: google/text-bison@002
+    tokenizer_name: google/text-bison@002
+    max_sequence_length: 6000 # Officially 8192
+    max_sequence_and_generated_tokens_length: 9216
+    client_spec:
+      class_name: "helm.proxy.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+  - name: google/text-bison-32k
+    model_name: google/text-bison-32k
+    tokenizer_name: google/text-bison@001
+    max_sequence_length: 32000
+    max_sequence_and_generated_tokens_length: 32000
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+
+  - name: google/text-unicorn@001
+    model_name: google/text-unicorn@001
+    tokenizer_name: google/text-unicorn@001
+    max_sequence_length: 6000 # Officially 8192
+    max_sequence_and_generated_tokens_length: 7000 # Officially 9216
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+  - name: google/code-bison@001
+    model_name: google/code-bison@001
+    tokenizer_name: google/mt5-base # TODO #2188: change to actual tokenizer
+    max_sequence_length: 6000 # Officially 6144
+    max_sequence_and_generated_tokens_length: 7000 # Officially 7168
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+  - name: google/code-bison@002
+    model_name: google/code-bison@002
+    tokenizer_name: google/mt5-base # TODO #2188: change to actual tokenizer
+    max_sequence_length: 6000 # Officially 6144
+    max_sequence_and_generated_tokens_length: 7168
+    client_spec:
+      class_name: "helm.proxy.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+  - name: google/code-bison-32k
+    model_name: google/code-bison-32k
+    tokenizer_name: google/mt5-base # TODO #2188: change to actual tokenizer
+    max_sequence_length: 32000
+    max_sequence_and_generated_tokens_length: 32000
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+  # HuggingFace
+
+  ## Bigcode
+  - name: huggingface/santacoder
+    model_name: bigcode/santacoder
+    tokenizer_name: bigcode/santacoder
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/starcoder
+    model_name: bigcode/starcoder
+    tokenizer_name: bigcode/starcoder
+    max_sequence_length: 8192
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  ## Databricks
+  - name: huggingface/dolly-v2-3b
+    model_name: databricks/dolly-v2-3b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/dolly-v2-7b
+    model_name: databricks/dolly-v2-7b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/dolly-v2-12b
+    model_name: databricks/dolly-v2-12b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  ## EleutherAI
+  - name: huggingface/pythia-1b-v0
+    model_name: eleutherai/pythia-1b-v0
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/pythia-2.8b-v0
+    model_name: eleutherai/pythia-2.8b-v0
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/pythia-6.9b
+    model_name: eleutherai/pythia-6.9b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/pythia-12b-v0
+    model_name: eleutherai/pythia-12b-v0
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/gpt-j-6b
+    model_name: eleutherai/gpt-j-6b
+    tokenizer_name: EleutherAI/gpt-j-6B
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/gpt-neox-20b
+    model_name: eleutherai/gpt-neox-20b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  ## LMSYS
+  - name: huggingface/vicuna-7b-v1.3
+    model_name: lmsys/vicuna-7b-v1.3
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/vicuna-13b-v1.3
+    model_name: lmsys/vicuna-13b-v1.3
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  ## Meta
+  - name: huggingface/opt-175b
+    model_name: meta/opt-175b
+    tokenizer_name: facebook/opt-66b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: facebook/opt-175b
+
+  - name: huggingface/opt-66b
+    model_name: meta/opt-66b
+    tokenizer_name: facebook/opt-66b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: facebook/opt-66b
+
+  - name: huggingface/opt-6.7b
+    model_name: meta/opt-6.7b
+    tokenizer_name: facebook/opt-66b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: facebook/opt-6.7b
+
+  - name: huggingface/opt-1.3b
+    model_name: meta/opt-1.3b
+    tokenizer_name: facebook/opt-66b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: facebook/opt-1.3b
+
+  ## Microsoft
+  - name: huggingface/llava-1.5-7b-hf
+    model_name: microsoft/llava-1.5-7b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+  
+  - name: huggingface/llava-1.5-13b-hf
+    model_name: microsoft/llava-1.5-13b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+      
+  ## OpenFlamingo
+  - name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
+    model_name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
+    tokenizer_name: anas-awadalla/mpt-7b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.open_flamingo_client.OpenFlamingoClient"
+      args:
+        checkpoint_path: "openflamingo/OpenFlamingo-9B-vitl-mpt7b"
+        tokenizer_name: "anas-awadalla-2/mpt-7b"
+        cross_attn_every_n_layers: 4
+
+  - name: together/phi-2
+    model_name: microsoft/phi-2
+    tokenizer_name: microsoft/phi-2
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient" 
+
+  ## Mistral AI
+  - name: huggingface/bakLlava-v1-hf
+    model_name: mistralai/bakLlava-v1-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
+  ## MosaicML
+  - name: huggingface/mpt-7b
+    model_name: mosaicml/mpt-7b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: mosaicml/mpt-7b
+
+  - name: huggingface/mpt-instruct-7b
+    model_name: mosaicml/mpt-instruct-7b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: mosaicml/mpt-7b-instruct
+
+  - name: huggingface/mpt-30b
+    model_name: mosaicml/mpt-30b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/mpt-instruct-30b
+    model_name: mosaicml/mpt-instruct-30b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: mosaicml/mpt-30b-instruct
+
+  ## OpenAI
+  - name: huggingface/gpt2
+    model_name: openai/gpt2
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 1024
+    max_request_length: 1025
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: openai-community/gpt2
+
+  ## StabilityAI
+  - name: huggingface/stablelm-base-alpha-3b
+    model_name: stabilityai/stablelm-base-alpha-3b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/stablelm-base-alpha-7b
+    model_name: stabilityai/stablelm-base-alpha-7b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  ## Text-to-Image Diffusion Models
+
+  - name: huggingface/dreamlike-diffusion-v1-0
+    model_name: huggingface/dreamlike-diffusion-v1-0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/dreamlike-photoreal-v2-0
+    model_name: huggingface/dreamlike-photoreal-v2-0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/openjourney-v1-0
+    model_name: huggingface/openjourney-v1-0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/openjourney-v2-0
+    model_name: huggingface/openjourney-v2-0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/redshift-diffusion
+    model_name: huggingface/redshift-diffusion
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/promptist-stable-diffusion-v1-4
+    model_name: huggingface/promptist-stable-diffusion-v1-4
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-v1-4
+    model_name: huggingface/stable-diffusion-v1-4
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-v1-5
+    model_name: huggingface/stable-diffusion-v1-5
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-v2-base
+    model_name: huggingface/stable-diffusion-v2-base
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-v2-1-base
+    model_name: huggingface/stable-diffusion-v2-1-base
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-safe-weak
+    model_name: huggingface/stable-diffusion-safe-weak
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-safe-medium
+    model_name: huggingface/stable-diffusion-safe-medium
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-safe-strong
+    model_name: huggingface/stable-diffusion-safe-strong
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-safe-max
+    model_name: huggingface/stable-diffusion-safe-max
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/vintedois-diffusion-v0-1
+    model_name: huggingface/vintedois-diffusion-v0-1
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: segmind/Segmind-Vega
+    model_name: segmind/Segmind-Vega
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: segmind/SSD-1B
+    model_name: segmind/SSD-1B
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: stabilityai/stable-diffusion-xl-base-1.0
+    model_name: stabilityai/stable-diffusion-xl-base-1.0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  # HuggingFaceM4
+  - name: HuggingFaceM4/idefics-9b
+    model_name: HuggingFaceM4/idefics-9b
+    tokenizer_name: HuggingFaceM4/idefics-9b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"
+
+  - name: HuggingFaceM4/idefics-9b-instruct
+    model_name: HuggingFaceM4/idefics-9b-instruct
+    tokenizer_name: HuggingFaceM4/idefics-9b-instruct
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"
+
+  - name: HuggingFaceM4/idefics-80b
+    model_name: HuggingFaceM4/idefics-80b
+    tokenizer_name: HuggingFaceM4/idefics-80b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"
+
+  - name: HuggingFaceM4/idefics-80b-instruct
+    model_name: HuggingFaceM4/idefics-80b-instruct
+    tokenizer_name: HuggingFaceM4/idefics-80b-instruct
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"
+
+  # Lexica
+  - name: lexica/search-stable-diffusion-1.5
+    model_name: lexica/search-stable-diffusion-1.5
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 200
+    client_spec:
+      class_name: "helm.clients.image_generation.lexica_client.LexicaClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.lexica_search_window_service.LexicaSearchWindowService"
+
+  # Kakao
+  - name: kakaobrain/mindall-e
+    model_name: kakaobrain/mindall-e
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.mindalle_client.MinDALLEClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  # Lighting AI
+  - name: lightningai/lit-gpt
+    model_name: lightningai/lit-gpt
+    tokenizer_name: lightningai/lit-gpt
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.lit_gpt_client.LitGPTClient"
+      args:
+        checkpoint_dir: "" # Path to the checkpoint directory
+        precision: bf16-true
+
+  # Mistral AI
+  - name: mistralai/mistral-tiny
+    model_name: mistralai/mistral-7b-v0.1
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.mistral_client.MistralAIClient"
+      args:
+        mistral_model: "mistral-tiny"
+
+  - name: mistralai/mistral-small-2402
+    model_name: mistralai/mistral-small-2402
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.mistral_client.MistralAIClient"
+
+  - name: mistralai/mistral-medium-2312
+    model_name: mistralai/mistral-medium-2312
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.mistral_client.MistralAIClient"
+
+  - name: mistralai/mistral-large-2402
+    model_name: mistralai/mistral-large-2402
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.mistral_client.MistralAIClient"
+
+  # Neurips
+  - name: neurips/local
+    model_name: neurips/local
+    tokenizer_name: neurips/local
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.http_model_client.HTTPModelClient"
+
+  # Nvidia
+  - name: nvidia/megatron-gpt2
+    model_name: nvidia/megatron-gpt2
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 1024
+    client_spec:
+      class_name: "helm.clients.megatron_client.MegatronClient"
+
+  # OpenAI
+
+  ## GPT 3 Models
+
+  - name: openai/davinci-002
+    model_name: openai/davinci-002
+    tokenizer_name: openai/cl100k_base
+    # Claimed sequence length is 16,384 tokens but we round down to 16,000 tokens
+    # to provide a margin of error.
+    max_sequence_length: 16000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/babbage-002
+    model_name: openai/babbage-002
+    tokenizer_name: openai/cl100k_base
+    # Claimed sequence length is 16,384 tokens but we round down to 16,000 tokens
+    # to provide a margin of error.
+    max_sequence_length: 16000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  # The list of models can be found here: https://beta.openai.com/docs/engines/gpt-3
+  # DEPRECATED: Announced on July 06 2023 that these models will be shut down on January 04 2024.
+
+  - name: openai/davinci
+    deprecated: true
+    model_name: openai/davinci
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/curie
+    deprecated: true
+    model_name: openai/curie
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/babbage
+    deprecated: true
+    model_name: openai/babbage
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/ada
+    deprecated: true
+    model_name: openai/ada
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/text-davinci-003
+    deprecated: true
+    model_name: openai/text-davinci-003
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 4000
+    max_request_length: 4001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/text-davinci-002
+    deprecated: true
+    model_name: openai/text-davinci-002
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 4000
+    max_request_length: 4001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/text-davinci-001
+    deprecated: true
+    model_name: openai/text-davinci-001
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/text-curie-001
+    deprecated: true
+    model_name: openai/text-curie-001
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/text-babbage-001
+    deprecated: true
+    model_name: openai/text-babbage-001
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/text-ada-001
+    deprecated: true
+    model_name: openai/text-ada-001
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  ## GPT 3.5 Turbo Models
+  # ChatGPT: https://openai.com/blog/chatgpt
+
+  - name: openai/gpt-3.5-turbo-instruct
+    model_name: openai/gpt-3.5-turbo-instruct
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 4096
+    max_request_length: 4097
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  # The claimed sequence length is 4096, but as of 2023-03-07, the empirical usable
+  # sequence length is smaller at 4087 with one user input message and one assistant
+  # output message because ChatGPT uses special tokens for message roles and boundaries.
+  # We use a rounded-down sequence length of 4000 to account for these special tokens.
+  - name: openai/gpt-3.5-turbo-0301
+    model_name: openai/gpt-3.5-turbo-0301
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 4000
+    max_request_length: 4001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  # The claimed sequence length is 4096, but as of 2023-03-07, the empirical usable
+  # sequence length is smaller at 4087 with one user input message and one assistant
+  # output message because ChatGPT uses special tokens for message roles and boundaries.
+  # We use a rounded-down sequence length of 4000 to account for these special tokens.
+  - name: openai/gpt-3.5-turbo-0613
+    model_name: openai/gpt-3.5-turbo-0613
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 4000
+    max_request_length: 4001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  # Claimed length is 16,384; we round down to 16,000 for the same reasons as explained
+  # in the openai/gpt-3.5-turbo-0613 comment
+  - name: openai/gpt-3.5-turbo-16k-0613
+    model_name: openai/gpt-3.5-turbo-16k-0613
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 16000
+    max_request_length: 16001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  # Claimed length is 16,384; we round down to 16,000 for the same reasons as explained
+  # in the openai/gpt-3.5-turbo-0613 comment
+  - name: openai/gpt-3.5-turbo-1106
+    model_name: openai/gpt-3.5-turbo-1106
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 16000
+    max_request_length: 16001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  # Claimed length is 16,384; we round down to 16,000 for the same reasons as explained
+  # in the openai/gpt-3.5-turbo-0613 comment
+  - name: openai/gpt-3.5-turbo-0125
+    model_name: openai/gpt-3.5-turbo-0125
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 16000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  ## GPT 4 Models
+
+  - name: openai/gpt-4-1106-preview
+    model_name: openai/gpt-4-1106-preview
+    tokenizer_name: openai/cl100k_base
+    # According to https://help.openai.com/en/articles/8555510-gpt-4-turbo,
+    # the maximum number of output tokens for this model is 4096
+    # TODO: add max_generated_tokens_length of 4096 https://github.com/stanford-crfm/helm/issues/2098
+    max_sequence_length: 128000
+    max_request_length: 128001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4-0314
+    model_name: openai/gpt-4-0314
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 8192
+    max_request_length: 8193
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4-32k-0314
+    model_name: openai/gpt-4-32k-0314
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 32768
+    max_request_length: 32769
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4-0613
+    model_name: openai/gpt-4-0613
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 8192
+    max_request_length: 8193
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4-32k-0613
+    model_name: openai/gpt-4-32k-0613
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 32768
+    max_request_length: 32769
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4-0125-preview
+    model_name: openai/gpt-4-0125-preview
+    tokenizer_name: openai/cl100k_base
+    # According to https://help.openai.com/en/articles/8555510-gpt-4-turbo,
+    # the maximum number of output tokens for this model is 4096
+    # TODO: add max_generated_tokens_length of 4096 https://github.com/stanford-crfm/helm/issues/2098
+    max_sequence_length: 128000
+    max_request_length: 128001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4-turbo-2024-04-09
+    model_name: openai/gpt-4-turbo-2024-04-09
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4-vision-preview
+    model_name: openai/gpt-4-vision-preview
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000  # According to https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
+    max_request_length: 128001
+    max_sequence_and_generated_tokens_length: 132096
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  ## Codex Models
+  # DEPRECATED: Codex models have been shut down on March 23 2023.
+
+  - name: openai/code-davinci-002
+    deprecated: true
+    model_name: openai/code-davinci-002
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 4000
+    max_request_length: 4001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/code-davinci-001
+    deprecated: true
+    model_name: openai/code-davinci-001
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/code-cushman-001
+    deprecated: true
+    model_name: openai/code-cushman-001
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  ## Text Similarity Models
+  # OpenAI similarity embedding models: https://beta.openai.com/docs/guides/embeddings
+  # The number of parameters is guessed based on the number of parameters of the
+  # corresponding GPT-3 model.
+  # DEPRECATED: Announced on July 06 2023 that first generation embeddings models
+  #  will be shut down on January 04 2024.
+
+  - name: openai/text-similarity-davinci-001
+    deprecated: true
+    model_name: openai/text-similarity-davinci-001
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/text-similarity-curie-001
+    deprecated: true
+    model_name: openai/text-similarity-curie-001
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/text-similarity-babbage-001
+    deprecated: true
+    model_name: openai/text-similarity-babbage-001
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/text-similarity-ada-001
+    deprecated: true
+    model_name: openai/text-similarity-ada-001
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  # As of 2023-11-07, text-embedding-ada-002 is not deprecated:
+  # "We recommend using text-embedding-ada-002 for nearly all use cases."
+  # Source: https://platform.openai.com/docs/guides/embeddings/what-are-embeddings
+  - name: openai/text-embedding-ada-002
+    model_name: openai/text-embedding-ada-002
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  # Text-to-image models
+  - name: openai/dall-e-2
+    model_name: openai/dall-e-2
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 1000
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle2_client.DALLE2Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"
+
+  - name: openai/dall-e-3
+    model_name: openai/dall-e-3
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 1000
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"
+
+  - name: openai/dall-e-3-natural
+    model_name: openai/dall-e-3-natural
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 1000
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"
+
+  - name: openai/dall-e-3-hd
+    model_name: openai/dall-e-3-hd
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 1000
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"
+
+  - name: openai/dall-e-3-hd-natural
+    model_name: openai/dall-e-3-hd-natural
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 1000
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"
+
+  # Together
+  # The list of models served by Together changes often, to check the latest list, visit:
+  # https://docs.together.ai/docs/inference-models
+  # You can also check the playground to check that the live models are working:
+  # https://api.together.xyz/playground
+
+  ## BigScience
+  - name: together/bloom
+    deprecated: true  # Removed from Together
+    model_name: bigscience/bloom
+    tokenizer_name: bigscience/bloom
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/t0pp
+    deprecated: true  # Removed from Together
+    model_name: bigscience/t0pp
+    tokenizer_name: bigscience/T0pp
+    max_sequence_length: 1024
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"
+
+  ## Google
+  - name: together/t5-11b
+    deprecated: true  # Removed from Together
+    model_name: google/t5-11b
+    tokenizer_name: google/t5-11b
+    max_sequence_length: 511
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"
+
+  - name: together/flan-t5-xxl
+    deprecated: true  # Removed from Together
+    model_name: google/flan-t5-xxl
+    tokenizer_name: google/flan-t5-xxl
+    max_sequence_length: 511
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"
+
+  - name: together/ul2
+    deprecated: true  # Removed from Together
+    model_name: google/ul2
+    tokenizer_name: google/ul2
+    max_sequence_length: 511
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"
+
+  ## Meta
+  - name: together/llama-7b
+    model_name: meta/llama-7b
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: huggyllama/llama-7b
+
+  - name: together/llama-13b
+    model_name: meta/llama-13b
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: huggyllama/llama-13b
+
+  - name: together/llama-30b
+    model_name: meta/llama-30b
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: huggyllama/llama-30b
+
+  - name: together/llama-65b
+    model_name: meta/llama-65b
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2047 # Subtract 1 tokens to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: huggyllama/llama-65b
+
+  - name: together/llama-2-7b
+    model_name: meta/llama-2-7b
+    tokenizer_name: meta-llama/Llama-2-7b-hf
+    max_sequence_length: 4094 # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/llama-2-7b
+
+  - name: together/llama-2-13b
+    model_name: meta/llama-2-13b
+    tokenizer_name: meta-llama/Llama-2-7b-hf
+    max_sequence_length: 4094 # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/llama-2-13b
+
+  - name: together/llama-2-70b
+    model_name: meta/llama-2-70b
+    tokenizer_name: meta-llama/Llama-2-7b-hf
+    max_sequence_length: 4094 # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/llama-2-70b
+
+  - name: together/llama-3-8b
+    model_name: meta/llama-3-8b
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-8B
+
+  - name: together/llama-3-70b
+    model_name: meta/llama-3-70b
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-70B
+
+  # 01.AI
+  - name: together/yi-6b
+    model_name: 01-ai/yi-6b
+    tokenizer_name: 01-ai/Yi-6B
+    max_sequence_length: 4095
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: zero-one-ai/Yi-6B
+
+  - name: together/yi-34b
+    model_name: 01-ai/yi-34b
+    tokenizer_name: 01-ai/Yi-6B
+    max_sequence_length: 4095
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: zero-one-ai/Yi-34B
+
+
+  # Allen Institute for AI
+  - name: together/olmo-7b
+    model_name: allenai/olmo-7b
+    tokenizer_name: allenai/olmo-7b
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/olmo-7b-twin-2t
+    model_name: allenai/olmo-7b-twin-2t
+    tokenizer_name: allenai/olmo-7b
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/olmo-7b-instruct
+    model_name: allenai/olmo-7b-instruct
+    tokenizer_name: allenai/olmo-7b
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+
+  ## MistralAI
+  - name: together/mistral-7b-v0.1
+    model_name: mistralai/mistral-7b-v0.1
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 4095 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: mistralai/Mistral-7B-v0.1
+
+  - name: together/mixtral-8x7b-32kseqlen
+    model_name: mistralai/mixtral-8x7b-32kseqlen
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 4095 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: mistralai/mixtral-8x7b-32kseqlen
+
+  - name: together/mixtral-8x7b-instruct-v0.1
+    model_name: mistralai/mixtral-8x7b-instruct-v0.1
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 4095 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/mixtral-8x22b
+    model_name: mistralai/mixtral-8x22b
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 65535
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  ## Stanford
+  - name: together/alpaca-7b
+    model_name: stanford/alpaca-7b
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/alpaca-7b
+
+  ## Tiiuae
+  - name: together/falcon-7b
+    model_name: tiiuae/falcon-7b
+    tokenizer_name: tiiuae/falcon-7b
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/falcon-7b
+
+  - name: together/falcon-7b-instruct
+    model_name: tiiuae/falcon-7b-instruct
+    tokenizer_name: tiiuae/falcon-7b
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/falcon-7b-instruct
+
+  - name: together/falcon-40b
+    model_name: tiiuae/falcon-40b
+    tokenizer_name: tiiuae/falcon-7b
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/falcon-40b
+
+  - name: together/falcon-40b-instruct
+    model_name: tiiuae/falcon-40b-instruct
+    tokenizer_name: tiiuae/falcon-7b
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/falcon-40b-instruct
+
+  ## Together
+  # These are models fine-tuned by Together (and not simply hosted by Together).
+  - name: together/gpt-jt-6b-v1
+    model_name: together/gpt-jt-6b-v1
+    tokenizer_name: EleutherAI/gpt-j-6B
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/GPT-JT-6B-v1
+
+  - name: together/gpt-neoxt-chat-base-20b
+    model_name: together/gpt-neoxt-chat-base-20b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/GPT-NeoXT-Chat-Base-20B
+
+  - name: together/redpajama-incite-base-3b-v1
+    model_name: together/redpajama-incite-base-3b-v1
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/RedPajama-INCITE-Base-3B-v1
+
+  - name: together/redpajama-incite-instruct-3b-v1
+    model_name: together/redpajama-incite-instruct-3b-v1
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/RedPajama-INCITE-Instruct-3B-v1
+
+  - name: together/redpajama-incite-base-7b
+    model_name: together/redpajama-incite-base-7b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/RedPajama-INCITE-7B-Base
+
+  - name: together/redpajama-incite-instruct-7b
+    model_name: together/redpajama-incite-instruct-7b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/RedPajama-INCITE-7B-Instruct
+
+  ## Tsinghua
+  - name: together/glm
+    deprecated: true  # Removed from Together
+    model_name: tsinghua/glm
+    tokenizer_name: TsinghuaKEG/ice
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.ice_window_service.ICEWindowService"
+
+  - name: thudm/cogview2
+    model_name: thudm/cogview2
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.cogview2_client.CogView2Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  ## Yandex
+  - name: together/yalm
+    deprecated: true  # Removed from Together
+    model_name: yandex/yalm
+    tokenizer_name: Yandex/yalm
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.yalm_window_service.YaLMWindowService"
+
+  # Writer
+  - name: writer/palmyra-base
+    model_name: writer/palmyra-base
+    tokenizer_name: writer/gpt2
+    max_sequence_length: 2048
+    max_sequence_and_generated_tokens_length: 2048
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
+
+  - name: writer/palmyra-large
+    model_name: writer/palmyra-large
+    tokenizer_name: writer/gpt2
+    max_sequence_length: 2048
+    max_sequence_and_generated_tokens_length: 2048
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
+
+  - name: writer/palmyra-instruct-30
+    model_name: writer/palmyra-instruct-30
+    tokenizer_name: writer/gpt2
+    max_sequence_length: 2048
+    max_sequence_and_generated_tokens_length: 2048
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
+
+  - name: writer/palmyra-e
+    model_name: writer/palmyra-e
+    tokenizer_name: writer/gpt2
+    max_sequence_length: 2048
+    max_sequence_and_generated_tokens_length: 2048
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
+
+  - name: writer/silk-road
+    model_name: writer/silk-road
+    tokenizer_name: writer/gpt2
+    max_sequence_length: 8192
+    max_sequence_and_generated_tokens_length: 8192
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
+
+  - name: writer/palmyra-x
+    model_name: writer/palmyra-x
+    tokenizer_name: writer/gpt2
+    max_sequence_length: 8192
+    max_sequence_and_generated_tokens_length: 8192
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
+
+  - name: writer/palmyra-x-v2
+    model_name: writer/palmyra-x-v2
+    tokenizer_name: writer/gpt2
+    max_sequence_length: 6000
+    max_sequence_and_generated_tokens_length: 7024
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
+
+  - name: writer/palmyra-x-v3
+    model_name: writer/palmyra-x-v3
+    tokenizer_name: writer/gpt2
+    max_sequence_length: 6000
+    max_sequence_and_generated_tokens_length: 7024
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
+
+  - name: writer/palmyra-x-32k
+    model_name: writer/palmyra-x-32k
+    tokenizer_name: writer/gpt2
+    max_sequence_length: 28000
+    max_sequence_and_generated_tokens_length: 30048
+    client_spec:
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
+
+  # Qwen
+
+  - name: together/qwen-7b
+    model_name: qwen/qwen-7b
+    tokenizer_name: qwen/qwen-7b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/Qwen-7B
+
+  - name: together/qwen1.5-7b
+    model_name: qwen/qwen1.5-7b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: Qwen/Qwen1.5-7B
+
+  - name: together/qwen1.5-14b
+    model_name: qwen/qwen1.5-14b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: Qwen/Qwen1.5-14B
+
+  - name: together/qwen1.5-32b
+    model_name: qwen/qwen1.5-32b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: Qwen/Qwen1.5-32B
+
+  - name: together/qwen1.5-72b
+    model_name: qwen/qwen1.5-72b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 4095
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: Qwen/Qwen1.5-72B
+
+  - name: huggingface/qwen-vl
+    model_name: qwen/qwen-vl
+    tokenizer_name: qwen/qwen-vl
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.vision_language.qwen_vlm_client.QwenVLMClient"
+
+  - name: huggingface/qwen-vl-chat
+    model_name: qwen/qwen-vl-chat
+    tokenizer_name: qwen/qwen-vl-chat
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.vision_language.qwen_vlm_client.QwenVLMClient"