crfm-helm 0.4.0__py3-none-any.whl → 0.5.1__py3-none-any.whl

helm/config/model_deployments.yaml

@@ -8,16 +8,23 @@
 #    # This file defines all the model deployments that you do not want to be public.
 #    model_deployments: [] # Leave empty to disable private model deployments
 
-
 model_deployments:
-
   - name: simple/model1
     model_name: simple/model1
-    tokenizer_name: simple/model1
+    tokenizer_name: simple/tokenizer1
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.simple_client.SimpleClient"
-      args: {}
+      class_name: "helm.clients.simple_client.SimpleClient"
+
+  # Adobe
+  - name: adobe/giga-gan
+    model_name: adobe/giga-gan
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.adobe_vision_client.AdobeVisionClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
 
   # AI21 Labs
 
@@ -29,14 +36,9 @@ model_deployments:
     tokenizer_name: ai21/j1
     max_sequence_length: 2047
     client_spec:
-      class_name: "helm.proxy.clients.ai21_client.AI21Client"
-      args: {}
+      class_name: "helm.clients.ai21_client.AI21Client"
     window_service_spec:
       class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-      args:
-        gpt2_window_service:
-          class_name: "helm.benchmark.window_services.gpt2_window_service.GPT2WindowService"
-          args: {}
 
   - name: ai21/j1-large
     deprecated: true
@@ -44,14 +46,9 @@ model_deployments:
     tokenizer_name: ai21/j1
     max_sequence_length: 2047
     client_spec:
-      class_name: "helm.proxy.clients.ai21_client.AI21Client"
-      args: {}
+      class_name: "helm.clients.ai21_client.AI21Client"
     window_service_spec:
       class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-      args:
-        gpt2_window_service:
-          class_name: "helm.benchmark.window_services.gpt2_window_service.GPT2WindowService"
-          args: {}
 
   - name: ai21/j1-grande
     deprecated: true
@@ -59,14 +56,9 @@ model_deployments:
     tokenizer_name: ai21/j1
     max_sequence_length: 2047
     client_spec:
-      class_name: "helm.proxy.clients.ai21_client.AI21Client"
-      args: {}
+      class_name: "helm.clients.ai21_client.AI21Client"
     window_service_spec:
       class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-      args:
-        gpt2_window_service:
-          class_name: "helm.benchmark.window_services.gpt2_window_service.GPT2WindowService"
-          args: {}
 
   - name: ai21/j1-grande-v2-beta
     deprecated: true
@@ -74,58 +66,36 @@ model_deployments:
     tokenizer_name: ai21/j1
     max_sequence_length: 2047
     client_spec:
-      class_name: "helm.proxy.clients.ai21_client.AI21Client"
-      args: {}
+      class_name: "helm.clients.ai21_client.AI21Client"
     window_service_spec:
       class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-      args:
-        gpt2_window_service:
-          class_name: "helm.benchmark.window_services.gpt2_window_service.GPT2WindowService"
-          args: {}
 
   - name: ai21/j2-jumbo
     model_name: ai21/j2-jumbo
     tokenizer_name: ai21/j1
     max_sequence_length: 6000
     client_spec:
-      class_name: "helm.proxy.clients.ai21_client.AI21Client"
-      args: {}
+      class_name: "helm.clients.ai21_client.AI21Client"
     window_service_spec:
-      class_name: "helm.benchmark.window_services.wider_ai21_window_service.AI21Jurassic2JumboWindowService"
-      args:
-        gpt2_window_service:
-          class_name: "helm.benchmark.window_services.gpt2_window_service.GPT2WindowService"
-          args: {}
+      class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
 
   - name: ai21/j2-large
     model_name: ai21/j2-large
     tokenizer_name: ai21/j1
     max_sequence_length: 2047
     client_spec:
-      class_name: "helm.proxy.clients.ai21_client.AI21Client"
-      args: {}
+      class_name: "helm.clients.ai21_client.AI21Client"
     window_service_spec:
       class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-      args:
-        gpt2_window_service:
-          class_name: "helm.benchmark.window_services.gpt2_window_service.GPT2WindowService"
-          args: {}
 
   - name: ai21/j2-grande
     model_name: ai21/j2-grande
     tokenizer_name: ai21/j1
     max_sequence_length: 2047
     client_spec:
-      class_name: "helm.proxy.clients.ai21_client.AI21Client"
-      args: {}
+      class_name: "helm.clients.ai21_client.AI21Client"
     window_service_spec:
       class_name: "helm.benchmark.window_services.ai21_window_service.AI21WindowService"
-      args:
-        gpt2_window_service:
-          class_name: "helm.benchmark.window_services.gpt2_window_service.GPT2WindowService"
-          args: {}
-
-
 
   # Aleph Alpha
   - name: AlephAlpha/luminous-base
@@ -133,29 +103,57 @@ model_deployments:
     tokenizer_name: AlephAlpha/luminous-base
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.aleph_alpha_client.AlephAlphaClient"
-      args: {}
+      class_name: "helm.clients.aleph_alpha_client.AlephAlphaClient"
 
   - name: AlephAlpha/luminous-extended
     model_name: AlephAlpha/luminous-extended
     tokenizer_name: AlephAlpha/luminous-extended
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.aleph_alpha_client.AlephAlphaClient"
-      args: {}
+      class_name: "helm.clients.aleph_alpha_client.AlephAlphaClient"
 
   - name: AlephAlpha/luminous-supreme
     model_name: AlephAlpha/luminous-supreme
     tokenizer_name: AlephAlpha/luminous-supreme
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.aleph_alpha_client.AlephAlphaClient"
-      args: {}
+      class_name: "helm.clients.aleph_alpha_client.AlephAlphaClient"
 
-  # TODO: Add luminous-world once it is released.
+  # TODO: Add luminous-world once it is released
+
+  - name: AlephAlpha/m-vader
+    model_name: AlephAlpha/m-vader
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.aleph_alpha_image_generation_client.AlephAlphaImageGenerationClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+
+  # Amazon
+  - name: amazon/titan-text-lite-v1
+    model_name: amazon/titan-text-lite-v1
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 4000
+    client_spec:
+      class_name: "helm.clients.bedrock_client.BedrockTitanClient"
+
+  - name: amazon/titan-tg1-large
+    model_name: amazon/titan-tg1-large
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 8000
+    client_spec:
+      class_name: "helm.clients.bedrock_client.BedrockTitanClient"
+
+  - name: amazon/titan-text-express-v1
+    model_name: amazon/titan-text-express-v1
+    tokenizer_name: huggingface/gpt2
+    max_sequence_length: 8000
+    client_spec:
+      class_name: "helm.clients.bedrock_client.BedrockTitanClient"
 
 
- 
   # Anthropic
   - name: anthropic/claude-v1.3
     model_name: anthropic/claude-v1.3
@@ -163,8 +161,7 @@ model_deployments:
     max_sequence_length: 8000
     max_sequence_and_generated_tokens_length: 9016
     client_spec:
-      class_name: "helm.proxy.clients.anthropic_client.AnthropicClient"
-      args: {}
+      class_name: "helm.clients.anthropic_client.AnthropicClient"
 
   - name: anthropic/claude-instant-v1
     model_name: anthropic/claude-instant-v1
@@ -172,8 +169,7 @@ model_deployments:
     max_sequence_length: 8000
     max_sequence_and_generated_tokens_length: 9016
     client_spec:
-      class_name: "helm.proxy.clients.anthropic_client.AnthropicClient"
-      args: {}
+      class_name: "helm.clients.anthropic_client.AnthropicClient"
 
   - name: anthropic/claude-instant-1.2
     model_name: anthropic/claude-instant-1.2
@@ -181,8 +177,7 @@ model_deployments:
     max_sequence_length: 8000
     max_sequence_and_generated_tokens_length: 9016
     client_spec:
-      class_name: "helm.proxy.clients.anthropic_client.AnthropicClient"
-      args: {}
+      class_name: "helm.clients.anthropic_client.AnthropicClient"
 
   - name: anthropic/claude-2.0
     model_name: anthropic/claude-2.0
@@ -190,8 +185,7 @@ model_deployments:
     max_sequence_length: 8000
     max_sequence_and_generated_tokens_length: 9016
     client_spec:
-      class_name: "helm.proxy.clients.anthropic_client.AnthropicClient"
-      args: {}
+      class_name: "helm.clients.anthropic_client.AnthropicClient"
 
   - name: anthropic/claude-2.1
     model_name: anthropic/claude-2.1
@@ -199,8 +193,28 @@ model_deployments:
     max_sequence_length: 8000
     max_sequence_and_generated_tokens_length: 9016
     client_spec:
-      class_name: "helm.proxy.clients.anthropic_client.AnthropicClient"
-      args: {}
+      class_name: "helm.clients.anthropic_client.AnthropicClient"
+
+  - name: anthropic/claude-3-sonnet-20240229
+    model_name: anthropic/claude-3-sonnet-20240229
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+
+  - name: anthropic/claude-3-haiku-20240307
+    model_name: anthropic/claude-3-haiku-20240307
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
+
+  - name: anthropic/claude-3-opus-20240229
+    model_name: anthropic/claude-3-opus-20240229
+    tokenizer_name: anthropic/claude
+    max_sequence_length: 200000
+    client_spec:
+      class_name: "helm.clients.anthropic_client.AnthropicMessagesClient"
 
   - name: anthropic/stanford-online-all-v4-s3
     deprecated: true # Closed model, not accessible via API
@@ -208,8 +222,7 @@ model_deployments:
     tokenizer_name: huggingface/gpt2
     max_sequence_length: 8192
     client_spec:
-      class_name: "helm.proxy.clients.anthropic_client.AnthropicLegacyClient"
-      args: {}
+      class_name: "helm.clients.anthropic_client.AnthropicLegacyClient"
 
   # Cohere
   - name: cohere/xlarge-20220609
@@ -218,11 +231,9 @@ model_deployments:
     max_sequence_length: 2047
     max_request_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.cohere_client.CohereClient"
-      args: {}
+      class_name: "helm.clients.cohere_client.CohereClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-      args: {}
 
   - name: cohere/large-20220720
     model_name: cohere/large-20220720
@@ -230,11 +241,9 @@ model_deployments:
     max_sequence_length: 2047
     max_request_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.cohere_client.CohereClient"
-      args: {}
+      class_name: "helm.clients.cohere_client.CohereClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-      args: {}
 
   - name: cohere/medium-20220720
     model_name: cohere/medium-20220720
@@ -242,11 +251,9 @@ model_deployments:
     max_sequence_length: 2047
     max_request_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.cohere_client.CohereClient"
-      args: {}
+      class_name: "helm.clients.cohere_client.CohereClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-      args: {}
 
   - name: cohere/small-20220720
     model_name: cohere/small-20220720
@@ -254,11 +261,9 @@ model_deployments:
     max_sequence_length: 2047
     max_request_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.cohere_client.CohereClient"
-      args: {}
+      class_name: "helm.clients.cohere_client.CohereClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-      args: {}
 
   - name: cohere/xlarge-20221108
     model_name: cohere/xlarge-20221108
@@ -266,11 +271,9 @@ model_deployments:
     max_sequence_length: 2047
     max_request_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.cohere_client.CohereClient"
-      args: {}
+      class_name: "helm.clients.cohere_client.CohereClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-      args: {}
 
   - name: cohere/medium-20221108
     model_name: cohere/medium-20221108
@@ -278,11 +281,9 @@ model_deployments:
     max_sequence_length: 2047
     max_request_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.cohere_client.CohereClient"
-      args: {}
+      class_name: "helm.clients.cohere_client.CohereClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
-      args: {}
 
   - name: cohere/command-medium-beta
     model_name: cohere/command-medium-beta
@@ -290,11 +291,9 @@ model_deployments:
     max_sequence_length: 2019
     max_request_length: 2020
     client_spec:
-      class_name: "helm.proxy.clients.cohere_client.CohereClient"
-      args: {}
+      class_name: "helm.clients.cohere_client.CohereClient"
     window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
-      args: {}
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
 
   - name: cohere/command-xlarge-beta
     model_name: cohere/command-xlarge-beta
@@ -302,11 +301,9 @@ model_deployments:
     max_sequence_length: 2019
     max_request_length: 2020
     client_spec:
-      class_name: "helm.proxy.clients.cohere_client.CohereClient"
-      args: {}
+      class_name: "helm.clients.cohere_client.CohereClient"
     window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
-      args: {}
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
 
   - name: cohere/command
     model_name: cohere/command
@@ -314,11 +311,9 @@ model_deployments:
     max_sequence_length: 2019 # TODO: verify this
     max_request_length: 2020 # TODO: verify this
     client_spec:
-      class_name: "helm.proxy.clients.cohere_client.CohereClient"
-      args: {}
+      class_name: "helm.clients.cohere_client.CohereClient"
     window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
-      args: {}
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
 
   - name: cohere/command-light
     model_name: cohere/command-light
@@ -326,38 +321,171 @@ model_deployments:
     max_sequence_length: 2019 # TODO: verify this
     max_request_length: 2020 # TODO: verify this
     client_spec:
-      class_name: "helm.proxy.clients.cohere_client.CohereClient"
-      args: {}
+      class_name: "helm.clients.cohere_client.CohereClient"
     window_service_spec:
-      class_name: "helm.benchmark.window_services.cohere_window_service.CohereCommandWindowService"
-      args: {}
+      class_name: "helm.benchmark.window_services.cohere_window_service.CohereWindowService"
+
+  # Craiyon
+
+  - name: craiyon/dalle-mini
+    model_name: craiyon/dalle-mini
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle_mini_client.DALLEMiniClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: craiyon/dalle-mega
+    model_name: craiyon/dalle-mega
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle_mini_client.DALLEMiniClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  # Databricks
+
+  - name: together/dbrx-instruct
+    model_name: databricks/dbrx-instruct
+    tokenizer_name: databricks/dbrx-instruct
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
 
+  # DeepFloyd
 
+  - name: DeepFloyd/IF-I-M-v1.0
+    model_name: DeepFloyd/IF-I-M-v1.0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.deep_floyd_client.DeepFloydClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: DeepFloyd/IF-I-L-v1.0
+    model_name: DeepFloyd/IF-I-L-v1.0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.deep_floyd_client.DeepFloydClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: DeepFloyd/IF-I-XL-v1.0
+    model_name: DeepFloyd/IF-I-XL-v1.0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.deep_floyd_client.DeepFloydClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  # Deepseek
+
+  - name: together/deepseek-llm-67b-chat
+    model_name: deepseek-ai/deepseek-llm-67b-chat
+    tokenizer_name: deepseek-ai/deepseek-llm-67b-chat
+    max_sequence_length: 4095
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
 
   # Gooseai
 
+  # TODO: Migrate these models to use OpenAIClient
+
   ## EleutherAI
-  - name: gooseai/gpt-neo-20b
-    model_name: eleutherai/gpt-neox-20b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
+  # - name: gooseai/gpt-neo-20b
+  #   model_name: eleutherai/gpt-neox-20b
+  #   tokenizer_name: EleutherAI/gpt-neox-20b
+  #   max_sequence_length: 2048
+  #   max_request_length: 2049
+  #   client_spec:
+  #     class_name: "helm.clients.goose_ai_client.GooseAIClient"
+
+  # - name: gooseai/gpt-j-6b
+  #   model_name: eleutherai/gpt-j-6b
+  #   tokenizer_name: EleutherAI/gpt-j-6B
+  #   max_sequence_length: 2048
+  #   max_request_length: 2049
+  #   client_spec:
+  #     class_name: "helm.clients.goose_ai_client.GooseAIClient"
+
+  # Google
+  # See: https://cloud.google.com/vertex-ai/docs/generative-ai/learn/model-versioning
+
+  ## Gemini
+  # See: https://ai.google.dev/models/gemini#model_variations
+  - name: google/gemini-pro
+    model_name: google/gemini-pro
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 30720
+    max_sequence_and_generated_tokens_length: 32768 # Officially max_sequence_length + 2048
     client_spec:
-      class_name: "helm.proxy.clients.goose_ai_client.GooseAIClient"
-      args: {}
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
 
-  - name: gooseai/gpt-j-6b
-    model_name: eleutherai/gpt-j-6b
-    tokenizer_name: EleutherAI/gpt-j-6B
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: google/gemini-1.0-pro-001
+    model_name: google/gemini-1.0-pro-001
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 30720
+    max_sequence_and_generated_tokens_length: 32768 # Officially max_sequence_length + 2048
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
+
+  - name: google/gemini-pro-vision
+    model_name: google/gemini-pro-vision
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 12288
+    max_sequence_and_generated_tokens_length: 16384 # Officially max_sequence_length + 4096, in practice max_output_tokens <= 2048 for vision models
     client_spec:
-      class_name: "helm.proxy.clients.goose_ai_client.GooseAIClient"
-      args: {}
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
 
+  - name: google/gemini-1.0-pro-vision-001
+    model_name: google/gemini-1.0-pro-vision-001
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 12288
+    max_sequence_and_generated_tokens_length: 16384
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
 
+  - name: google/gemini-1.5-pro-preview-0409
+    model_name: google/gemini-1.5-pro-preview-0409
+    tokenizer_name: google/gemma-2b  # Gemini has no tokenizer endpoint, so we approximate by using Gemma's tokenizer.
+    max_sequence_length: 1000000  # Source: https://cloud.google.com/vertex-ai/generative-ai/docs/learn/models#gemini-models
+    # TODO: Max output tokens: 8192
+    client_spec:
+      class_name: "helm.clients.vertexai_client.VertexAIChatClient"
 
-  # Google
+  ## Gemma
+  - name: together/gemma-2b
+    model_name: google/gemma-2b
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 7167
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/gemma-2b-it
+    model_name: google/gemma-2b-it
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 7167
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/gemma-7b
+    model_name: google/gemma-7b
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 7167
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/gemma-7b-it
+    model_name: google/gemma-7b-it
+    tokenizer_name: google/gemma-2b
+    max_sequence_length: 7167
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
 
   ## PaLM 2
   - name: google/text-bison@001
@@ -366,20 +494,30 @@ model_deployments:
     max_sequence_length: 6000 # Officially 8192
     max_sequence_and_generated_tokens_length: 7000 # Officially 9216
     client_spec:
-      class_name: "helm.proxy.clients.vertexai_client.VertexAIClient"
-      args: {}
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
 
+  - name: google/text-bison@002
+    model_name: google/text-bison@002
+    tokenizer_name: google/text-bison@002
+    max_sequence_length: 6000 # Officially 8192
+    max_sequence_and_generated_tokens_length: 9216
+    client_spec:
+      class_name: "helm.proxy.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
 
   - name: google/text-bison-32k
     model_name: google/text-bison-32k
-    tokenizer_name: google/mt5-base
+    tokenizer_name: google/text-bison@001
     max_sequence_length: 32000
     max_sequence_and_generated_tokens_length: 32000
     client_spec:
-      class_name: "helm.proxy.clients.vertexai_client.VertexAIClient"
-      args: {}
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
 
   - name: google/text-unicorn@001
     model_name: google/text-unicorn@001
@@ -387,30 +525,39 @@ model_deployments:
     max_sequence_length: 6000 # Officially 8192
     max_sequence_and_generated_tokens_length: 7000 # Officially 9216
     client_spec:
-      class_name: "helm.proxy.clients.vertexai_client.VertexAIClient"
-      args: {}
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
 
   - name: google/code-bison@001
     model_name: google/code-bison@001
-    tokenizer_name: google/mt5-base
+    tokenizer_name: google/mt5-base # TODO #2188: change to actual tokenizer
     max_sequence_length: 6000 # Officially 6144
     max_sequence_and_generated_tokens_length: 7000 # Officially 7168
     client_spec:
-      class_name: "helm.proxy.clients.vertexai_client.VertexAIClient"
-      args: {}
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
+
+  - name: google/code-bison@002
+    model_name: google/code-bison@002
+    tokenizer_name: google/mt5-base # TODO #2188: change to actual tokenizer
+    max_sequence_length: 6000 # Officially 6144
+    max_sequence_and_generated_tokens_length: 7168
+    client_spec:
+      class_name: "helm.proxy.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
 
   - name: google/code-bison-32k
     model_name: google/code-bison-32k
-    tokenizer_name: google/mt5-base
+    tokenizer_name: google/mt5-base # TODO #2188: change to actual tokenizer
     max_sequence_length: 32000
     max_sequence_and_generated_tokens_length: 32000
     client_spec:
-      class_name: "helm.proxy.clients.vertexai_client.VertexAIClient"
-      args: {}
-
-
+      class_name: "helm.clients.vertexai_client.VertexAITextClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.no_decoding_window_service.NoDecodingWindowService"
 
   # HuggingFace
 
@@ -420,26 +567,238 @@ model_deployments:
     tokenizer_name: bigcode/santacoder
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.huggingface_client.HuggingFaceClient"
-      args: {}
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
 
   - name: huggingface/starcoder
     model_name: bigcode/starcoder
     tokenizer_name: bigcode/starcoder
     max_sequence_length: 8192
     client_spec:
-      class_name: "helm.proxy.clients.huggingface_client.HuggingFaceClient"
-      args: {}
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  ## Databricks
+  - name: huggingface/dolly-v2-3b
+    model_name: databricks/dolly-v2-3b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/dolly-v2-7b
+    model_name: databricks/dolly-v2-7b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/dolly-v2-12b
+    model_name: databricks/dolly-v2-12b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
 
   ## EleutherAI
+  - name: huggingface/pythia-1b-v0
+    model_name: eleutherai/pythia-1b-v0
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/pythia-2.8b-v0
+    model_name: eleutherai/pythia-2.8b-v0
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/pythia-6.9b
+    model_name: eleutherai/pythia-6.9b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/pythia-12b-v0
+    model_name: eleutherai/pythia-12b-v0
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
   - name: huggingface/gpt-j-6b
     model_name: eleutherai/gpt-j-6b
     tokenizer_name: EleutherAI/gpt-j-6B
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.huggingface_client.HuggingFaceClient"
-      args: {}
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/gpt-neox-20b
+    model_name: eleutherai/gpt-neox-20b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    max_request_length: 2049
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  ## LMSYS
+  - name: huggingface/vicuna-7b-v1.3
+    model_name: lmsys/vicuna-7b-v1.3
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/vicuna-13b-v1.3
+    model_name: lmsys/vicuna-13b-v1.3
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  ## Meta
+  - name: huggingface/opt-175b
+    model_name: meta/opt-175b
+    tokenizer_name: facebook/opt-66b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: facebook/opt-175b
+
+  - name: huggingface/opt-66b
+    model_name: meta/opt-66b
+    tokenizer_name: facebook/opt-66b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: facebook/opt-66b
+
+  - name: huggingface/opt-6.7b
+    model_name: meta/opt-6.7b
+    tokenizer_name: facebook/opt-66b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: facebook/opt-6.7b
+
+  - name: huggingface/opt-1.3b
+    model_name: meta/opt-1.3b
+    tokenizer_name: facebook/opt-66b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: facebook/opt-1.3b
+
+  ## Microsoft
+  - name: huggingface/llava-1.5-7b-hf
+    model_name: microsoft/llava-1.5-7b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+  
+  - name: huggingface/llava-1.5-13b-hf
+    model_name: microsoft/llava-1.5-13b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
+  - name: huggingface/llava-v1.6-vicuna-7b-hf
+    model_name: uw-madison/llava-v1.6-vicuna-7b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
+  - name: huggingface/llava-v1.6-vicuna-13b-hf
+    model_name: uw-madison/llava-v1.6-vicuna-13b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
+  - name: huggingface/llava-v1.6-mistral-7b-hf
+    model_name: uw-madison/llava-v1.6-mistral-7b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
+  - name: huggingface/llava-v1.6-34b-hf
+    model_name: uw-madison/llava-v1.6-34b-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
+  ## OpenFlamingo
+  - name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
+    model_name: openflamingo/OpenFlamingo-9B-vitl-mpt7b
+    tokenizer_name: anas-awadalla/mpt-7b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.open_flamingo_client.OpenFlamingoClient"
+      args:
+        checkpoint_path: "openflamingo/OpenFlamingo-9B-vitl-mpt7b"
+        tokenizer_name: "anas-awadalla-2/mpt-7b"
+        cross_attn_every_n_layers: 4
+
+  - name: together/phi-2
+    model_name: microsoft/phi-2
+    tokenizer_name: microsoft/phi-2
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient" 
+
+  ## Mistral AI
+  - name: huggingface/bakLlava-v1-hf
+    model_name: mistralai/bakLlava-v1-hf
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vlm_client.HuggingFaceVLMClient"
+
+  ## MosaicML
+  - name: huggingface/mpt-7b
+    model_name: mosaicml/mpt-7b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: mosaicml/mpt-7b
+
+  - name: huggingface/mpt-instruct-7b
+    model_name: mosaicml/mpt-instruct-7b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: mosaicml/mpt-7b-instruct
+
+  - name: huggingface/mpt-30b
+    model_name: mosaicml/mpt-30b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/mpt-instruct-30b
+    model_name: mosaicml/mpt-instruct-30b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: mosaicml/mpt-30b-instruct
 
   ## OpenAI
   - name: huggingface/gpt2
@@ -448,43 +807,246 @@ model_deployments:
     max_sequence_length: 1024
     max_request_length: 1025
     client_spec:
-      class_name: "helm.proxy.clients.huggingface_client.HuggingFaceClient"
-      args: {}
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+      args:
+        pretrained_model_name_or_path: openai-community/gpt2
+
+  ## StabilityAI
+  - name: huggingface/stablelm-base-alpha-3b
+    model_name: stabilityai/stablelm-base-alpha-3b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  - name: huggingface/stablelm-base-alpha-7b
+    model_name: stabilityai/stablelm-base-alpha-7b
+    tokenizer_name: EleutherAI/gpt-neox-20b
+    max_sequence_length: 4096
+    client_spec:
+      class_name: "helm.clients.huggingface_client.HuggingFaceClient"
+
+  ## Text-to-Image Diffusion Models
+
+  - name: huggingface/dreamlike-diffusion-v1-0
+    model_name: huggingface/dreamlike-diffusion-v1-0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/dreamlike-photoreal-v2-0
+    model_name: huggingface/dreamlike-photoreal-v2-0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/openjourney-v1-0
+    model_name: huggingface/openjourney-v1-0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/openjourney-v2-0
+    model_name: huggingface/openjourney-v2-0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/redshift-diffusion
+    model_name: huggingface/redshift-diffusion
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/promptist-stable-diffusion-v1-4
+    model_name: huggingface/promptist-stable-diffusion-v1-4
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-v1-4
+    model_name: huggingface/stable-diffusion-v1-4
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-v1-5
+    model_name: huggingface/stable-diffusion-v1-5
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-v2-base
+    model_name: huggingface/stable-diffusion-v2-base
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-v2-1-base
+    model_name: huggingface/stable-diffusion-v2-1-base
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-safe-weak
+    model_name: huggingface/stable-diffusion-safe-weak
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-safe-medium
+    model_name: huggingface/stable-diffusion-safe-medium
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-safe-strong
+    model_name: huggingface/stable-diffusion-safe-strong
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/stable-diffusion-safe-max
+    model_name: huggingface/stable-diffusion-safe-max
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: huggingface/vintedois-diffusion-v0-1
+    model_name: huggingface/vintedois-diffusion-v0-1
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: segmind/Segmind-Vega
+    model_name: segmind/Segmind-Vega
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: segmind/SSD-1B
+    model_name: segmind/SSD-1B
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
+
+  - name: stabilityai/stable-diffusion-xl-base-1.0
+    model_name: stabilityai/stable-diffusion-xl-base-1.0
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.huggingface_diffusers_client.HuggingFaceDiffusersClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
 
   # HuggingFaceM4
+  - name: HuggingFaceM4/idefics2-8b
+    model_name: HuggingFaceM4/idefics2-8b
+    # From https://huggingface.co/docs/transformers/main/en/model_doc/idefics2,
+    # "constructs a IDEFICS2 processor which wraps a LLama tokenizer."
+    tokenizer_name: hf-internal-testing/llama-tokenizer
+    max_sequence_length: 2048
+    client_spec:
+      class_name: "helm.clients.vision_language.huggingface_vision2seq_client.HuggingFaceVision2SeqClient"
+
   - name: HuggingFaceM4/idefics-9b
     model_name: HuggingFaceM4/idefics-9b
     tokenizer_name: HuggingFaceM4/idefics-9b
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"
-      args: {}
+      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"
 
   - name: HuggingFaceM4/idefics-9b-instruct
     model_name: HuggingFaceM4/idefics-9b-instruct
     tokenizer_name: HuggingFaceM4/idefics-9b-instruct
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"
-      args: {}
+      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"
 
   - name: HuggingFaceM4/idefics-80b
     model_name: HuggingFaceM4/idefics-80b
     tokenizer_name: HuggingFaceM4/idefics-80b
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"
-      args: {}
+      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"
 
   - name: HuggingFaceM4/idefics-80b-instruct
     model_name: HuggingFaceM4/idefics-80b-instruct
     tokenizer_name: HuggingFaceM4/idefics-80b-instruct
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.vision_language.idefics_client.IDEFICSClient"
-      args: {}
+      class_name: "helm.clients.vision_language.idefics_client.IDEFICSClient"
 
+  # Lexica
+  - name: lexica/search-stable-diffusion-1.5
+    model_name: lexica/search-stable-diffusion-1.5
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 200
+    client_spec:
+      class_name: "helm.clients.image_generation.lexica_client.LexicaClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.lexica_search_window_service.LexicaSearchWindowService"
 
+  # Kakao
+  - name: kakaobrain/mindall-e
+    model_name: kakaobrain/mindall-e
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.mindalle_client.MinDALLEClient"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
 
   # Lighting AI
   - name: lightningai/lit-gpt
@@ -492,44 +1054,49 @@ model_deployments:
     tokenizer_name: lightningai/lit-gpt
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.lit_gpt_client.LitGPTClient"
+      class_name: "helm.clients.lit_gpt_client.LitGPTClient"
       args:
         checkpoint_dir: "" # Path to the checkpoint directory
         precision: bf16-true
 
-
-
-  # Microsoft
-  - name: microsoft/TNLGv2_530B
-    model_name: microsoft/TNLGv2_530B
-    tokenizer_name: microsoft/gpt2
-    max_sequence_length: 2047
-    max_request_length: 2048
+  # Mistral AI
+  - name: mistralai/mistral-tiny
+    model_name: mistralai/mistral-7b-v0.1
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 32000
     client_spec:
-      class_name: "helm.proxy.clients.microsoft_client.MicrosoftClient"
-      args: {}
+      class_name: "helm.clients.mistral_client.MistralAIClient"
+      args:
+        mistral_model: "mistral-tiny"
 
-  - name: microsoft/TNLGv2_7B
-    model_name: microsoft/TNLGv2_7B
-    tokenizer_name: microsoft/gpt2
-    max_sequence_length: 2047
-    max_request_length: 2048
+  - name: mistralai/mistral-small-2402
+    model_name: mistralai/mistral-small-2402
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 32000
     client_spec:
-      class_name: "helm.proxy.clients.microsoft_client.MicrosoftClient"
-      args: {}
+      class_name: "helm.clients.mistral_client.MistralAIClient"
 
+  - name: mistralai/mistral-medium-2312
+    model_name: mistralai/mistral-medium-2312
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.mistral_client.MistralAIClient"
 
+  - name: mistralai/mistral-large-2402
+    model_name: mistralai/mistral-large-2402
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 32000
+    client_spec:
+      class_name: "helm.clients.mistral_client.MistralAIClient"
 
   # Neurips
   - name: neurips/local
     model_name: neurips/local
     tokenizer_name: neurips/local
     max_sequence_length: 2048
-    client_spec:
-      class_name: "helm.proxy.clients.http_model_client.HTTPModelClient"
-      args: {}
-
-
+    client_spec:
+      class_name: "helm.clients.http_model_client.HTTPModelClient"
 
   # Nvidia
   - name: nvidia/megatron-gpt2
@@ -537,17 +1104,33 @@ model_deployments:
     tokenizer_name: huggingface/gpt2
     max_sequence_length: 1024
     client_spec:
-      class_name: "helm.proxy.clients.megatron_client.MegatronClient"
-      args: {}
-
-
+      class_name: "helm.clients.megatron_client.MegatronClient"
 
   # OpenAI
 
   ## GPT 3 Models
+
+  - name: openai/davinci-002
+    model_name: openai/davinci-002
+    tokenizer_name: openai/cl100k_base
+    # Claimed sequence length is 16,384 tokens but we round down to 16,000 tokens
+    # to provide a margin of error.
+    max_sequence_length: 16000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/babbage-002
+    model_name: openai/babbage-002
+    tokenizer_name: openai/cl100k_base
+    # Claimed sequence length is 16,384 tokens but we round down to 16,000 tokens
+    # to provide a margin of error.
+    max_sequence_length: 16000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
   # The list of models can be found here: https://beta.openai.com/docs/engines/gpt-3
   # DEPRECATED: Announced on July 06 2023 that these models will be shut down on January 04 2024.
- 
+
   - name: openai/davinci
     deprecated: true
     model_name: openai/davinci
@@ -555,8 +1138,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/curie
     deprecated: true
@@ -565,8 +1147,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/babbage
     deprecated: true
@@ -575,8 +1156,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/ada
     deprecated: true
@@ -585,8 +1165,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/text-davinci-003
     deprecated: true
@@ -595,8 +1174,7 @@ model_deployments:
     max_sequence_length: 4000
     max_request_length: 4001
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/text-davinci-002
     deprecated: true
@@ -605,8 +1183,7 @@ model_deployments:
     max_sequence_length: 4000
     max_request_length: 4001
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/text-davinci-001
     deprecated: true
@@ -615,8 +1192,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/text-curie-001
     deprecated: true
@@ -625,8 +1201,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/text-babbage-001
     deprecated: true
@@ -635,8 +1210,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/text-ada-001
     deprecated: true
@@ -645,13 +1219,19 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
-
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   ## GPT 3.5 Turbo Models
   # ChatGPT: https://openai.com/blog/chatgpt
 
+  - name: openai/gpt-3.5-turbo-instruct
+    model_name: openai/gpt-3.5-turbo-instruct
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 4096
+    max_request_length: 4097
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
   # The claimed sequence length is 4096, but as of 2023-03-07, the empirical usable
   # sequence length is smaller at 4087 with one user input message and one assistant
   # output message because ChatGPT uses special tokens for message roles and boundaries.
@@ -662,8 +1242,7 @@ model_deployments:
     max_sequence_length: 4000
     max_request_length: 4001
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   # The claimed sequence length is 4096, but as of 2023-03-07, the empirical usable
   # sequence length is smaller at 4087 with one user input message and one assistant
@@ -675,8 +1254,7 @@ model_deployments:
     max_sequence_length: 4000
     max_request_length: 4001
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   # Claimed length is 16,384; we round down to 16,000 for the same reasons as explained
   # in the openai/gpt-3.5-turbo-0613 comment
@@ -686,9 +1264,26 @@ model_deployments:
     max_sequence_length: 16000
     max_request_length: 16001
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  # Claimed length is 16,384; we round down to 16,000 for the same reasons as explained
+  # in the openai/gpt-3.5-turbo-0613 comment
+  - name: openai/gpt-3.5-turbo-1106
+    model_name: openai/gpt-3.5-turbo-1106
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 16000
+    max_request_length: 16001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
+  # Claimed length is 16,384; we round down to 16,000 for the same reasons as explained
+  # in the openai/gpt-3.5-turbo-0613 comment
+  - name: openai/gpt-3.5-turbo-0125
+    model_name: openai/gpt-3.5-turbo-0125
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 16000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   ## GPT 4 Models
 
@@ -701,8 +1296,7 @@ model_deployments:
     max_sequence_length: 128000
     max_request_length: 128001
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/gpt-4-0314
     model_name: openai/gpt-4-0314
@@ -710,8 +1304,7 @@ model_deployments:
     max_sequence_length: 8192
     max_request_length: 8193
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/gpt-4-32k-0314
     model_name: openai/gpt-4-32k-0314
@@ -719,8 +1312,7 @@ model_deployments:
     max_sequence_length: 32768
     max_request_length: 32769
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/gpt-4-0613
     model_name: openai/gpt-4-0613
@@ -728,8 +1320,7 @@ model_deployments:
     max_sequence_length: 8192
     max_request_length: 8193
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/gpt-4-32k-0613
     model_name: openai/gpt-4-32k-0613
@@ -737,9 +1328,43 @@ model_deployments:
     max_sequence_length: 32768
     max_request_length: 32769
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4-0125-preview
+    model_name: openai/gpt-4-0125-preview
+    tokenizer_name: openai/cl100k_base
+    # According to https://help.openai.com/en/articles/8555510-gpt-4-turbo,
+    # the maximum number of output tokens for this model is 4096
+    # TODO: add max_generated_tokens_length of 4096 https://github.com/stanford-crfm/helm/issues/2098
+    max_sequence_length: 128000
+    max_request_length: 128001
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4-turbo-2024-04-09
+    model_name: openai/gpt-4-turbo-2024-04-09
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  - name: openai/gpt-4-vision-preview
+    model_name: openai/gpt-4-vision-preview
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000  # According to https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
+    max_request_length: 128001
+    max_sequence_and_generated_tokens_length: 132096
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
+  - name: openai/gpt-4-1106-vision-preview
+    model_name: openai/gpt-4-1106-vision-preview
+    tokenizer_name: openai/cl100k_base
+    max_sequence_length: 128000  # According to https://platform.openai.com/docs/models/gpt-4-and-gpt-4-turbo
+    max_request_length: 128001
+    max_sequence_and_generated_tokens_length: 132096
+    client_spec:
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   ## Codex Models
   # DEPRECATED: Codex models have been shut down on March 23 2023.
@@ -751,8 +1376,7 @@ model_deployments:
     max_sequence_length: 4000
     max_request_length: 4001
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/code-davinci-001
     deprecated: true
@@ -761,8 +1385,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/code-cushman-001
     deprecated: true
@@ -771,10 +1394,8 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
- 
   ## Text Similarity Models
   # OpenAI similarity embedding models: https://beta.openai.com/docs/guides/embeddings
   # The number of parameters is guessed based on the number of parameters of the
@@ -789,8 +1410,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/text-similarity-curie-001
     deprecated: true
@@ -799,8 +1419,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/text-similarity-babbage-001
     deprecated: true
@@ -809,8 +1428,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   - name: openai/text-similarity-ada-001
     deprecated: true
@@ -819,8 +1437,7 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
 
   # As of 2023-11-07, text-embedding-ada-002 is not deprecated:
   # "We recommend using text-embedding-ada-002 for nearly all use cases."
@@ -831,10 +1448,53 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.openai_client.OpenAIClient"
-      args: {}
+      class_name: "helm.clients.openai_client.OpenAIClient"
+
+  # Text-to-image models
+  - name: openai/dall-e-2
+    model_name: openai/dall-e-2
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 1000
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle2_client.DALLE2Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"
+
+  - name: openai/dall-e-3
+    model_name: openai/dall-e-3
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 1000
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"
+
+  - name: openai/dall-e-3-natural
+    model_name: openai/dall-e-3-natural
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 1000
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"
 
+  - name: openai/dall-e-3-hd
+    model_name: openai/dall-e-3-hd
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 1000
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"
 
+  - name: openai/dall-e-3-hd-natural
+    model_name: openai/dall-e-3-hd-natural
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 1000
+    client_spec:
+      class_name: "helm.clients.image_generation.dalle3_client.DALLE3Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.openai_dalle_window_service.OpenAIDALLEWindowService"
 
   # Together
   # The list of models served by Together changes often, to check the latest list, visit:
@@ -844,275 +1504,154 @@ model_deployments:
 
   ## BigScience
   - name: together/bloom
-    deprecated: true # Removed from together
+    deprecated: true  # Removed from Together
     model_name: bigscience/bloom
     tokenizer_name: bigscience/bloom
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
 
   - name: together/t0pp
-    deprecated: true # Removed from together
+    deprecated: true  # Removed from Together
     model_name: bigscience/t0pp
     tokenizer_name: bigscience/T0pp
     max_sequence_length: 1024
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
     window_service_spec:
-      class_name: "helm.benchmark.window_services.t0pp_window_service.T0ppWindowService"
-      args: {}
-
-  ## Databricks
-  - name: together/dolly-v2-3b
-    model_name: databricks/dolly-v2-3b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  - name: together/dolly-v2-7b
-    model_name: databricks/dolly-v2-7b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  - name: together/dolly-v2-12b
-    model_name: databricks/dolly-v2-12b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  ## EleutherAI
-  - name: together/gpt-j-6b
-    deprecated: true # Removed from together
-    model_name: eleutherai/gpt-j-6b
-    tokenizer_name: EleutherAI/gpt-j-6B
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  - name: together/gpt-neox-20b
-    deprecated: true # Removed from together
-    model_name: eleutherai/gpt-neox-20b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  - name: together/pythia-1b-v0
-    model_name: eleutherai/pythia-1b-v0
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  - name: together/pythia-2.8b-v0
-    model_name: eleutherai/pythia-2.8b-v0
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  - name: together/pythia-6.9b
-    model_name: eleutherai/pythia-6.9b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  - name: together/pythia-12b-v0
-    model_name: eleutherai/pythia-12b-v0
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"
 
   ## Google
   - name: together/t5-11b
-    deprecated: true # Removed from together
+    deprecated: true  # Removed from Together
     model_name: google/t5-11b
     tokenizer_name: google/t5-11b
     max_sequence_length: 511
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
     window_service_spec:
-      class_name: "helm.benchmark.window_services.t511b_window_service.T511bWindowService"
-      args: {}
+      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"
 
   - name: together/flan-t5-xxl
-    deprecated: true # Removed from together
+    deprecated: true  # Removed from Together
     model_name: google/flan-t5-xxl
     tokenizer_name: google/flan-t5-xxl
     max_sequence_length: 511
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
     window_service_spec:
-      class_name: "helm.benchmark.window_services.flan_t5_window_service.FlanT5WindowService"
-      args: {}
+      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"
 
   - name: together/ul2
-    deprecated: true # Removed from together
+    deprecated: true  # Removed from Together
     model_name: google/ul2
     tokenizer_name: google/ul2
     max_sequence_length: 511
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
     window_service_spec:
-      class_name: "helm.benchmark.window_services.ul2_window_service.UL2WindowService"
-      args: {}
-
-  ## HazyResearch
-  - name: together/h3-2.7b
-    deprecated: true # Not available on Together yet
-    model_name: hazyresearch/h3-2.7b
-    tokenizer_name: huggingface/gpt2
-    max_sequence_length: 1024
-    max_request_length: 1025
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  ## LMSYS
-  # TODO: might be deprecated. Needs to be checked.
-  # Together officialy supports vicuna 1.5, not sure if 1.3 is still supported.
-  - name: together/vicuna-7b-v1.3
-    model_name: lmsys/vicuna-7b-v1.3
-    tokenizer_name: hf-internal-testing/llama-tokenizer
-    max_sequence_length: 2048
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  - name: together/vicuna-13b-v1.3
-    model_name: lmsys/vicuna-13b-v1.3
-    tokenizer_name: hf-internal-testing/llama-tokenizer
-    max_sequence_length: 2048
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.benchmark.window_services.encoder_decoder_window_service.EncoderDecoderWindowService"
 
   ## Meta
   - name: together/llama-7b
     model_name: meta/llama-7b
     tokenizer_name: hf-internal-testing/llama-tokenizer
-    max_sequence_length: 2047  # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: huggyllama/llama-7b
 
   - name: together/llama-13b
     model_name: meta/llama-13b
     tokenizer_name: hf-internal-testing/llama-tokenizer
-    max_sequence_length: 2047  # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: huggyllama/llama-13b
 
   - name: together/llama-30b
     model_name: meta/llama-30b
     tokenizer_name: hf-internal-testing/llama-tokenizer
-    max_sequence_length: 2047  # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: huggyllama/llama-30b
 
   - name: together/llama-65b
     model_name: meta/llama-65b
     tokenizer_name: hf-internal-testing/llama-tokenizer
-    max_sequence_length: 2047  # Subtract 1 tokens to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 2047 # Subtract 1 tokens to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: huggyllama/llama-65b
 
   - name: together/llama-2-7b
     model_name: meta/llama-2-7b
     tokenizer_name: meta-llama/Llama-2-7b-hf
-    max_sequence_length: 4094  # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
+    max_sequence_length: 4094 # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/llama-2-7b
 
   - name: together/llama-2-13b
     model_name: meta/llama-2-13b
     tokenizer_name: meta-llama/Llama-2-7b-hf
-    max_sequence_length: 4094  # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
+    max_sequence_length: 4094 # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/llama-2-13b
 
   - name: together/llama-2-70b
     model_name: meta/llama-2-70b
     tokenizer_name: meta-llama/Llama-2-7b-hf
-    max_sequence_length: 4094  # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
+    max_sequence_length: 4094 # Subtract 2 tokens to work around a off-by-two bug in Together's token counting (#2080 and #2094)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/llama-2-70b
 
-  - name: together/opt-175b
-    deprecated: true # Not available on Together yet
-    model_name: meta/opt-175b
-    tokenizer_name: facebook/opt-66b
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: together/llama-3-8b
+    model_name: meta/llama-3-8b
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-8B
 
-  - name: together/opt-66b
-    deprecated: true # Not available on Together yet
-    model_name: meta/opt-66b
-    tokenizer_name: facebook/opt-66b
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: together/llama-3-70b
+    model_name: meta/llama-3-70b
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-70B
 
-  - name: together/opt-6.7b
-    deprecated: true # Not available on Together yet
-    model_name: meta/opt-6.7b
-    tokenizer_name: facebook/opt-66b
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: together/llama-3-8b-chat
+    model_name: meta/llama-3-8b-chat
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-8B
 
-  - name: together/opt-1.3b
-    deprecated: true # Not available on Together yet
-    model_name: meta/opt-1.3b
-    tokenizer_name: facebook/opt-66b
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: together/llama-3-70b-chat
+    model_name: meta/llama-3-70b-chat
+    tokenizer_name: meta/llama-3-8b
+    max_sequence_length: 8191
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: meta-llama/Meta-Llama-3-70B
 
   # 01.AI
   - name: together/yi-6b
@@ -1120,93 +1659,109 @@ model_deployments:
     tokenizer_name: 01-ai/Yi-6B
     max_sequence_length: 4095
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: zero-one-ai/Yi-6B
 
   - name: together/yi-34b
     model_name: 01-ai/yi-34b
     tokenizer_name: 01-ai/Yi-6B
     max_sequence_length: 4095
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: zero-one-ai/Yi-34B
+
+  - name: together/yi-6b-chat
+    model_name: 01-ai/yi-6b-chat
+    tokenizer_name: 01-ai/Yi-6B
+    max_sequence_length: 4095
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: zero-one-ai/Yi-6B
+
+  - name: together/yi-34b-chat
+    model_name: 01-ai/yi-34b-chat
+    tokenizer_name: 01-ai/Yi-6B
+    max_sequence_length: 4095
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: zero-one-ai/Yi-34B
+
+
+  # Allen Institute for AI
+  - name: together/olmo-7b
+    model_name: allenai/olmo-7b
+    tokenizer_name: allenai/olmo-7b
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/olmo-7b-twin-2t
+    model_name: allenai/olmo-7b-twin-2t
+    tokenizer_name: allenai/olmo-7b
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
+  - name: together/olmo-7b-instruct
+    model_name: allenai/olmo-7b-instruct
+    tokenizer_name: allenai/olmo-7b
+    max_sequence_length: 2047
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+
 
   ## MistralAI
   - name: together/mistral-7b-v0.1
     model_name: mistralai/mistral-7b-v0.1
     tokenizer_name: mistralai/Mistral-7B-v0.1
-    max_sequence_length: 4095  # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 4095 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: mistralai/Mistral-7B-v0.1
 
   - name: together/mixtral-8x7b-32kseqlen
     model_name: mistralai/mixtral-8x7b-32kseqlen
     tokenizer_name: mistralai/Mistral-7B-v0.1
-    max_sequence_length: 4095  # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
-
-  ## MosaicML
-  - name: together/mpt-7b
-    deprecated: true # Not available on Together yet
-    model_name: mosaicml/mpt-7b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
+    max_sequence_length: 4095 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: mistralai/mixtral-8x7b-32kseqlen
 
-  - name: together/mpt-instruct-7b
-    deprecated: true # Not available on Together yet
-    model_name: mosaicml/mpt-instruct-7b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: together/mixtral-8x7b-instruct-v0.1
+    model_name: mistralai/mixtral-8x7b-instruct-v0.1
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 4095 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
 
-  - name: together/mpt-30b
-    model_name: mosaicml/mpt-30b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: together/mixtral-8x22b
+    model_name: mistralai/mixtral-8x22b
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 65535
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
 
-  - name: together/mpt-instruct-30b
-    model_name: mosaicml/mpt-instruct-30b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 2048
-    max_request_length: 2049
+  - name: together/mixtral-8x22b-instruct-v0.1
+    model_name: mistralai/mixtral-8x22b-instruct-v0.1
+    tokenizer_name: mistralai/Mistral-7B-v0.1
+    max_sequence_length: 65535
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
 
-  ## StabilityAI
-  - name: together/stablelm-base-alpha-3b
-    deprecated: true # Removed from together
-    model_name: stabilityai/stablelm-base-alpha-3b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 4096
-    max_request_length: 4097
-    client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
 
-  - name: together/stablelm-base-alpha-7b
-    deprecated: true # Removed from together
-    model_name: stabilityai/stablelm-base-alpha-7b
-    tokenizer_name: EleutherAI/gpt-neox-20b
-    max_sequence_length: 4096
-    max_request_length: 4097
+  ## Snowflake
+  - name: together/snowflake-arctic-instruct
+    model_name: snowflake/snowflake-arctic-instruct
+    tokenizer_name: snowflake/snowflake-arctic-instruct
+    max_sequence_length: 4000  # Lower than 4096 because of chat tokens
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherChatClient"
 
   ## Stanford
   - name: together/alpaca-7b
@@ -1214,41 +1769,46 @@ model_deployments:
     tokenizer_name: hf-internal-testing/llama-tokenizer
     max_sequence_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/alpaca-7b
 
   ## Tiiuae
   - name: together/falcon-7b
     model_name: tiiuae/falcon-7b
     tokenizer_name: tiiuae/falcon-7b
-    max_sequence_length: 2047  # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/falcon-7b
 
   - name: together/falcon-7b-instruct
     model_name: tiiuae/falcon-7b-instruct
     tokenizer_name: tiiuae/falcon-7b
-    max_sequence_length: 2047  # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/falcon-7b-instruct
 
   - name: together/falcon-40b
     model_name: tiiuae/falcon-40b
     tokenizer_name: tiiuae/falcon-7b
-    max_sequence_length: 2047  # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/falcon-40b
 
   - name: together/falcon-40b-instruct
     model_name: tiiuae/falcon-40b-instruct
     tokenizer_name: tiiuae/falcon-7b
-    max_sequence_length: 2047  # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
+    max_sequence_length: 2047 # Subtract 1 token to work around a off-by-one bug in Together's input validation token counting (#2080)
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/falcon-40b-instruct
 
   ## Together
   # These are models fine-tuned by Together (and not simply hosted by Together).
@@ -1258,8 +1818,9 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/GPT-JT-6B-v1
 
   - name: together/gpt-neoxt-chat-base-20b
     model_name: together/gpt-neoxt-chat-base-20b
@@ -1267,8 +1828,9 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/GPT-NeoXT-Chat-Base-20B
 
   - name: together/redpajama-incite-base-3b-v1
     model_name: together/redpajama-incite-base-3b-v1
@@ -1276,8 +1838,9 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/RedPajama-INCITE-Base-3B-v1
 
   - name: together/redpajama-incite-instruct-3b-v1
     model_name: together/redpajama-incite-instruct-3b-v1
@@ -1285,8 +1848,9 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/RedPajama-INCITE-Instruct-3B-v1
 
   - name: together/redpajama-incite-base-7b
     model_name: together/redpajama-incite-base-7b
@@ -1294,8 +1858,9 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/RedPajama-INCITE-7B-Base
 
   - name: together/redpajama-incite-instruct-7b
     model_name: together/redpajama-incite-instruct-7b
@@ -1303,38 +1868,42 @@ model_deployments:
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/RedPajama-INCITE-7B-Instruct
 
   ## Tsinghua
   - name: together/glm
-    deprecated: true # Not available on Together yet
+    deprecated: true  # Removed from Together
     model_name: tsinghua/glm
     tokenizer_name: TsinghuaKEG/ice
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.ice_window_service.ICEWindowService"
-      args: {}
+
+  - name: thudm/cogview2
+    model_name: thudm/cogview2
+    tokenizer_name: openai/clip-vit-large-patch14
+    max_sequence_length: 75
+    client_spec:
+      class_name: "helm.clients.image_generation.cogview2_client.CogView2Client"
+    window_service_spec:
+      class_name: "helm.benchmark.window_services.image_generation.clip_window_service.CLIPWindowService"
 
   ## Yandex
   - name: together/yalm
-    deprecated: true # Not available on Together yet
+    deprecated: true  # Removed from Together
     model_name: yandex/yalm
     tokenizer_name: Yandex/yalm
     max_sequence_length: 2048
     max_request_length: 2049
     client_spec:
-      class_name: "helm.proxy.clients.together_client.TogetherClient"
-      args: {}
+      class_name: "helm.clients.together_client.TogetherClient"
     window_service_spec:
       class_name: "helm.benchmark.window_services.yalm_window_service.YaLMWindowService"
-      args: {}
-
-
 
   # Writer
   - name: writer/palmyra-base
@@ -1343,8 +1912,7 @@ model_deployments:
     max_sequence_length: 2048
     max_sequence_and_generated_tokens_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.palmyra_client.PalmyraClient"
-      args: {}
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
 
   - name: writer/palmyra-large
     model_name: writer/palmyra-large
@@ -1352,8 +1920,7 @@ model_deployments:
     max_sequence_length: 2048
     max_sequence_and_generated_tokens_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.palmyra_client.PalmyraClient"
-      args: {}
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
 
   - name: writer/palmyra-instruct-30
     model_name: writer/palmyra-instruct-30
@@ -1361,8 +1928,7 @@ model_deployments:
     max_sequence_length: 2048
     max_sequence_and_generated_tokens_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.palmyra_client.PalmyraClient"
-      args: {}
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
 
   - name: writer/palmyra-e
     model_name: writer/palmyra-e
@@ -1370,8 +1936,7 @@ model_deployments:
     max_sequence_length: 2048
     max_sequence_and_generated_tokens_length: 2048
     client_spec:
-      class_name: "helm.proxy.clients.palmyra_client.PalmyraClient"
-      args: {}
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
 
   - name: writer/silk-road
     model_name: writer/silk-road
@@ -1379,8 +1944,7 @@ model_deployments:
     max_sequence_length: 8192
     max_sequence_and_generated_tokens_length: 8192
     client_spec:
-      class_name: "helm.proxy.clients.palmyra_client.PalmyraClient"
-      args: {}
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
 
   - name: writer/palmyra-x
     model_name: writer/palmyra-x
@@ -1388,8 +1952,7 @@ model_deployments:
     max_sequence_length: 8192
     max_sequence_and_generated_tokens_length: 8192
     client_spec:
-      class_name: "helm.proxy.clients.palmyra_client.PalmyraClient"
-      args: {}
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
 
   - name: writer/palmyra-x-v2
     model_name: writer/palmyra-x-v2
@@ -1397,8 +1960,7 @@ model_deployments:
     max_sequence_length: 6000
     max_sequence_and_generated_tokens_length: 7024
     client_spec:
-      class_name: "helm.proxy.clients.palmyra_client.PalmyraClient"
-      args: {}
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
 
   - name: writer/palmyra-x-v3
     model_name: writer/palmyra-x-v3
@@ -1406,8 +1968,7 @@ model_deployments:
     max_sequence_length: 6000
     max_sequence_and_generated_tokens_length: 7024
     client_spec:
-      class_name: "helm.proxy.clients.palmyra_client.PalmyraClient"
-      args: {}
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
 
   - name: writer/palmyra-x-32k
     model_name: writer/palmyra-x-32k
@@ -1415,5 +1976,65 @@ model_deployments:
     max_sequence_length: 28000
     max_sequence_and_generated_tokens_length: 30048
     client_spec:
-      class_name: "helm.proxy.clients.palmyra_client.PalmyraClient"
-      args: {}
+      class_name: "helm.clients.palmyra_client.PalmyraClient"
+
+  # Qwen
+
+  - name: together/qwen-7b
+    model_name: qwen/qwen-7b
+    tokenizer_name: qwen/qwen-7b
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: togethercomputer/Qwen-7B
+
+  - name: together/qwen1.5-7b
+    model_name: qwen/qwen1.5-7b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: Qwen/Qwen1.5-7B
+
+  - name: together/qwen1.5-14b
+    model_name: qwen/qwen1.5-14b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: Qwen/Qwen1.5-14B
+
+  - name: together/qwen1.5-32b
+    model_name: qwen/qwen1.5-32b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 32767
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: Qwen/Qwen1.5-32B
+
+  - name: together/qwen1.5-72b
+    model_name: qwen/qwen1.5-72b
+    tokenizer_name: qwen/qwen1.5-7b
+    max_sequence_length: 4095
+    client_spec:
+      class_name: "helm.clients.together_client.TogetherClient"
+      args:
+        together_model: Qwen/Qwen1.5-72B
+
+  - name: huggingface/qwen-vl
+    model_name: qwen/qwen-vl
+    tokenizer_name: qwen/qwen-vl
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.vision_language.qwen_vlm_client.QwenVLMClient"
+
+  - name: huggingface/qwen-vl-chat
+    model_name: qwen/qwen-vl-chat
+    tokenizer_name: qwen/qwen-vl-chat
+    max_sequence_length: 8191
+    client_spec:
+      class_name: "helm.clients.vision_language.qwen_vlm_client.QwenVLMClient"