PyPI - infer-stack - Versions diffs - 0.6.0__py3-none-any.whl - Mend

infer-stack 0.6.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (44) hide show

infer_stack/__init__.py +2 -0
infer_stack/backends/__init__.py +7 -0
infer_stack/backends/compose_renderer.py +243 -0
infer_stack/backends/kubeai_renderer.py +202 -0
infer_stack/benchmark.py +38 -0
infer_stack/catalog.py +438 -0
infer_stack/cli/__init__.py +169 -0
infer_stack/cli/__main__.py +4 -0
infer_stack/cli/commands_profile.py +467 -0
infer_stack/cli/commands_runtime.py +719 -0
infer_stack/cli/commands_smoke.py +691 -0
infer_stack/cli/compose.py +755 -0
infer_stack/cli/context.py +471 -0
infer_stack/cli/options.py +134 -0
infer_stack/cli/probes.py +178 -0
infer_stack/config.py +450 -0
infer_stack/contracts.py +223 -0
infer_stack/diff_prompt.py +117 -0
infer_stack/docker_utils.py +230 -0
infer_stack/env_utils.py +97 -0
infer_stack/experimental/model_catalog_discover.py +1155 -0
infer_stack/experimental/model_memory_estimator.py +1264 -0
infer_stack/experimental/stress_test_long_context.py +397 -0
infer_stack/hardware.py +70 -0
infer_stack/kubeai_ops.py +76 -0
infer_stack/paths.py +87 -0
infer_stack/profile_runtime.py +46 -0
infer_stack/renderer.py +19 -0
infer_stack/resolver.py +1092 -0
infer_stack/templates/default-models.yaml +674 -0
infer_stack/templates/default-ollama-models.yaml +31 -0
infer_stack/templates/default-profiles.yaml +1731 -0
infer_stack/templates/default-vllm-models.yaml +714 -0
infer_stack/templates/docker-compose.yml.j2 +430 -0
infer_stack/templates/litellm_config.yaml.j2 +44 -0
infer_stack/templates/nginx.conf.j2 +84 -0
infer_stack/tuning.py +3 -0
infer_stack/validator.py +314 -0
infer_stack/verification.py +46 -0
infer_stack-0.6.0.dist-info/METADATA +1034 -0
infer_stack-0.6.0.dist-info/RECORD +44 -0
infer_stack-0.6.0.dist-info/WHEEL +5 -0
infer_stack-0.6.0.dist-info/entry_points.txt +2 -0
infer_stack-0.6.0.dist-info/top_level.txt +1 -0

infer_stack/templates/default-vllm-models.yaml ADDED Viewed

@@ -0,0 +1,714 @@
+vllm_models:
+  qwen3.5-0.8b:
+    hf_model_id: Qwen/Qwen3.5-0.8B
+    served_model_name: qwen3.5-0.8b
+    family: qwen3.5
+    modalities:
+    - text
+    memory_class_gib: 4
+    min_vram_gib_per_replica: 4
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 32
+  qwen3.5-2b:
+    hf_model_id: Qwen/Qwen3.5-2B
+    served_model_name: qwen3.5-2b
+    family: qwen3.5
+    modalities:
+    - text
+    memory_class_gib: 8
+    min_vram_gib_per_replica: 8
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 32
+  qwen3.5-4b:
+    hf_model_id: Qwen/Qwen3.5-4B
+    served_model_name: qwen3.5-4b
+    family: qwen3.5
+    modalities:
+    - text
+    memory_class_gib: 12
+    min_vram_gib_per_replica: 12
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 24
+  qwen3.5-9b:
+    hf_model_id: Qwen/Qwen3.5-9B
+    served_model_name: qwen3.5-9b
+    family: qwen3.5
+    modalities:
+    - text
+    memory_class_gib: 20
+    min_vram_gib_per_replica: 20
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  qwen3.5-27b:
+    hf_model_id: Qwen/Qwen3.5-27B
+    served_model_name: qwen3.5-27b
+    family: qwen3.5
+    modalities:
+    - text
+    memory_class_gib: 48
+    min_vram_gib_per_replica: 48
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  qwen3.5-35b-a3b:
+    hf_model_id: Qwen/Qwen3.5-35B-A3B
+    served_model_name: qwen3.5-35b-a3b
+    family: qwen3.5
+    modalities:
+    - text
+    memory_class_gib: 48
+    min_vram_gib_per_replica: 48
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  qwen3.5-122b-a10b:
+    hf_model_id: Qwen/Qwen3.5-122B-A10B
+    served_model_name: qwen3.5-122b-a10b
+    family: qwen3.5
+    modalities:
+    - text
+    memory_class_gib: 80
+    min_vram_gib_per_replica: 80
+    preferred_gpu_count: 2
+    context_window: 131072
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 8
+  qwen3.5-122b-a10b-fp8:
+    hf_model_id: Qwen/Qwen3.5-122B-A10B-FP8
+    served_model_name: qwen3.5-122b-a10b-fp8
+    family: qwen3.5
+    modalities:
+    - text
+    memory_class_gib: 80
+    min_vram_gib_per_replica: 80
+    preferred_gpu_count: 2
+    context_window: 262144
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 8
+    tool_calling:
+      auto: true
+      parser: qwen3_xml
+  qwen3.6-35b-a3b:
+    hf_model_id: Qwen/Qwen3.6-35B-A3B
+    served_model_name: qwen3.6-35b-a3b
+    tokenizer_name: Qwen/Qwen3.6-35B-A3B
+    family: qwen3.6
+    modalities:
+    - text
+    memory_class_gib: 80
+    min_vram_gib_per_replica: 24
+    preferred_gpu_count: 2
+    context_window: 262144
+    supported_protocols:
+    - chat
+    - completions
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 262144
+      gpu_memory_utilization: 0.95
+      enable_prefix_caching: false
+      max_num_batched_tokens: 8192
+      max_num_seqs: 8
+  qwen3.6-35b-a3b-fp8:
+    hf_model_id: Qwen/Qwen3.6-35B-A3B-FP8
+    served_model_name: qwen3.6-35b-a3b
+    tokenizer_name: Qwen/Qwen3.6-35B-A3B
+    family: qwen3.6
+    modalities:
+    - text
+    memory_class_gib: 40
+    min_vram_gib_per_replica: 24
+    preferred_gpu_count: 1
+    context_window: 262144
+    supported_protocols:
+    - chat
+    - completions
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 8192
+      gpu_memory_utilization: 0.92
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 2
+    tool_calling:
+      auto: true
+      parser: qwen3_coder
+  gemma4-e2b:
+    hf_model_id: google/gemma-4-E2B-it
+    served_model_name: gemma4-e2b
+    family: gemma4
+    modalities:
+    - text
+    - image
+    - audio
+    memory_class_gib: 8
+    min_vram_gib_per_replica: 8
+    preferred_gpu_count: 1
+    context_window: 131072
+    thinking_history_policy: keep_final_only
+    notes:
+    - Historical turns should keep only final responses, not prior thinking content.
+    defaults:
+      max_model_len: 65536
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 16
+    tool_calling:
+      auto: true
+      parser: gemma4
+  gemma4-e4b:
+    hf_model_id: google/gemma-4-E4B-it
+    served_model_name: gemma4-e4b
+    family: gemma4
+    modalities:
+    - text
+    - image
+    - audio
+    memory_class_gib: 10
+    min_vram_gib_per_replica: 10
+    preferred_gpu_count: 1
+    context_window: 131072
+    thinking_history_policy: keep_final_only
+    notes:
+    - Historical turns should keep only final responses, not prior thinking content.
+    defaults:
+      max_model_len: 65536
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 16
+    tool_calling:
+      auto: true
+      parser: gemma4
+  gemma4-26b:
+    hf_model_id: google/gemma-4-26B-A4B-it
+    served_model_name: gemma4-26b
+    family: gemma4
+    modalities:
+    - text
+    - image
+    memory_class_gib: 20
+    min_vram_gib_per_replica: 20
+    preferred_gpu_count: 1
+    context_window: 262144
+    thinking_history_policy: keep_final_only
+    notes:
+    - Historical turns should keep only final responses, not prior thinking content.
+    defaults:
+      max_model_len: 65536
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 12
+    tool_calling:
+      auto: true
+      parser: gemma4
+  gemma4-31b:
+    hf_model_id: google/gemma-4-31B-it
+    served_model_name: gemma4-31b
+    family: gemma4
+    modalities:
+    - text
+    - image
+    memory_class_gib: 22
+    min_vram_gib_per_replica: 22
+    preferred_gpu_count: 1
+    context_window: 262144
+    thinking_history_policy: keep_final_only
+    notes:
+    - Historical turns should keep only final responses, not prior thinking content.
+    defaults:
+      max_model_len: 65536
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 12
+    tool_calling:
+      auto: true
+      parser: gemma4
+  qwen2-72b-instruct:
+    hf_model_id: Qwen/Qwen2-72B-Instruct
+    served_model_name: qwen/qwen2-72b-instruct
+    logical_model_name: qwen/qwen2-72b-instruct
+    tokenizer_name: qwen/qwen2-72b-instruct
+    family: qwen2
+    modalities:
+    - text
+    memory_class_gib: 144
+    min_vram_gib_per_replica: 144
+    preferred_gpu_count: 2
+    context_window: 131072
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 8
+  qwen2.5-0.5b-instruct:
+    hf_model_id: Qwen/Qwen2.5-0.5B-Instruct
+    served_model_name: qwen/qwen2.5-0.5b-instruct
+    logical_model_name: qwen/qwen2.5-0.5b-instruct
+    tokenizer_name: Qwen/Qwen2.5-0.5B-Instruct
+    family: qwen2.5
+    modalities:
+    - text
+    memory_class_gib: 2
+    min_vram_gib_per_replica: 2
+    preferred_gpu_count: 1
+    context_window: 32768
+    defaults:
+      max_model_len: 8192
+      gpu_memory_utilization: 0.85
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 16
+  qwen2.5-7b-instruct:
+    hf_model_id: Qwen/Qwen2.5-7B-Instruct
+    served_model_name: qwen/qwen2.5-7b-instruct-turbo
+    logical_model_name: qwen/qwen2.5-7b-instruct-turbo
+    tokenizer_name: qwen/qwen2.5-7b-instruct-turbo
+    family: qwen2.5
+    modalities:
+    - text
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 131072
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  qwen2.5-72b-instruct:
+    hf_model_id: Qwen/Qwen2.5-72B-Instruct
+    served_model_name: qwen/qwen2.5-72b-instruct-turbo
+    logical_model_name: qwen/qwen2.5-72b-instruct-turbo
+    tokenizer_name: qwen/qwen2.5-72b-instruct-turbo
+    family: qwen2.5
+    modalities:
+    - text
+    memory_class_gib: 144
+    min_vram_gib_per_replica: 144
+    preferred_gpu_count: 2
+    context_window: 131072
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 8
+  gpt-oss-20b:
+    hf_model_id: openai/gpt-oss-20b
+    served_model_name: openai/gpt-oss-20b
+    logical_model_name: openai/gpt-oss-20b
+    tokenizer_name: openai/o200k_harmony
+    family: gpt-oss
+    modalities:
+    - text
+    memory_class_gib: 40
+    min_vram_gib_per_replica: 40
+    preferred_gpu_count: 1
+    context_window: 131072
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  llama-3.1-8b-instruct:
+    hf_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct
+    served_model_name: meta/llama-3.1-8b-instruct-turbo
+    family: llama3.1
+    modalities:
+    - text
+    memory_class_gib: 18
+    min_vram_gib_per_replica: 18
+    preferred_gpu_count: 1
+    context_window: 131072
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  llama-3-8b-instruct:
+    hf_model_id: meta-llama/Meta-Llama-3-8B-Instruct
+    served_model_name: meta/llama-3-8b-chat
+    family: llama3
+    modalities:
+    - text
+    memory_class_gib: 18
+    min_vram_gib_per_replica: 18
+    preferred_gpu_count: 1
+    context_window: 8192
+    defaults:
+      max_model_len: 8192
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  llama-2-7b:
+    hf_model_id: meta-llama/Llama-2-7b-hf
+    served_model_name: meta/llama-2-7b
+    family: llama2
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 4096
+    defaults:
+      max_model_len: 4096
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 16
+  llama-2-13b:
+    hf_model_id: meta-llama/Llama-2-13b-hf
+    served_model_name: meta/llama-2-13b
+    family: llama2
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 28
+    min_vram_gib_per_replica: 28
+    preferred_gpu_count: 1
+    context_window: 4096
+    defaults:
+      max_model_len: 4096
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 16
+  llama-2-70b:
+    hf_model_id: meta-llama/Llama-2-70b-hf
+    served_model_name: meta/llama-2-70b
+    family: llama2
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 140
+    min_vram_gib_per_replica: 70
+    preferred_gpu_count: 2
+    context_window: 4096
+    defaults:
+      max_model_len: 4096
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 8
+  mistral-7b-instruct-v0.3:
+    hf_model_id: mistralai/Mistral-7B-Instruct-v0.3
+    served_model_name: mistralai/mistral-7b-instruct-v0.3
+    family: mistral
+    modalities:
+    - text
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 32768
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  mistral-7b-v0.1:
+    hf_model_id: mistralai/Mistral-7B-v0.1
+    served_model_name: mistralai/mistral-7b-v0.1
+    family: mistral
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 32768
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  falcon-7b:
+    hf_model_id: tiiuae/falcon-7b
+    served_model_name: tiiuae/falcon-7b
+    family: falcon
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+  falcon-7b-instruct:
+    hf_model_id: tiiuae/falcon-7b-instruct
+    served_model_name: tiiuae/falcon-7b-instruct
+    family: falcon
+    modalities:
+    - text
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+  gpt2:
+    hf_model_id: openai-community/gpt2
+    served_model_name: gpt2
+    logical_model_name: gpt2
+    tokenizer_name: openai-community/gpt2
+    family: gpt2
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 1
+    min_vram_gib_per_replica: 1
+    preferred_gpu_count: 1
+    context_window: 1024
+    defaults:
+      max_model_len: 1024
+      gpu_memory_utilization: 0.3
+      enable_prefix_caching: true
+      max_num_batched_tokens: 1024
+      max_num_seqs: 16
+  smollm2-135m-instruct:
+    hf_model_id: HuggingFaceTB/SmolLM2-135M-Instruct
+    served_model_name: huggingfacetb/smollm2-135m-instruct
+    logical_model_name: huggingfacetb/smollm2-135m-instruct
+    tokenizer_name: HuggingFaceTB/SmolLM2-135M-Instruct
+    family: smollm2
+    modalities:
+    - text
+    memory_class_gib: 1
+    min_vram_gib_per_replica: 1
+    preferred_gpu_count: 1
+    context_window: 8192
+    defaults:
+      max_model_len: 4096
+      gpu_memory_utilization: 0.6
+      enable_prefix_caching: true
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+  pythia-160m:
+    hf_model_id: EleutherAI/pythia-160m
+    served_model_name: eleutherai/pythia-160m
+    logical_model_name: eleutherai/pythia-160m
+    tokenizer_name: EleutherAI/pythia-160m
+    family: pythia
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 1
+    min_vram_gib_per_replica: 1
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.6
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+  pythia-410m:
+    hf_model_id: EleutherAI/pythia-410m
+    served_model_name: eleutherai/pythia-410m
+    logical_model_name: eleutherai/pythia-410m
+    tokenizer_name: EleutherAI/pythia-410m
+    family: pythia
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 2
+    min_vram_gib_per_replica: 2
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.7
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+  pythia-6.9b:
+    hf_model_id: EleutherAI/pythia-6.9b
+    served_model_name: eleutherai/pythia-6.9b
+    logical_model_name: eleutherai/pythia-6.9b
+    tokenizer_name: eleutherai/pythia-6.9b
+    family: pythia
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+  pythia-2.8b-v0:
+    hf_model_id: EleutherAI/pythia-2.8b-v0
+    served_model_name: eleutherai/pythia-2.8b-v0
+    logical_model_name: eleutherai/pythia-2.8b-v0
+    tokenizer_name: eleutherai/pythia-2.8b-v0
+    family: pythia
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 8
+    min_vram_gib_per_replica: 8
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+  pythia-1b-v0:
+    hf_model_id: EleutherAI/pythia-1b-v0
+    served_model_name: eleutherai/pythia-1b-v0
+    logical_model_name: eleutherai/pythia-1b-v0
+    tokenizer_name: eleutherai/pythia-1b-v0
+    family: pythia
+    modalities:
+    - text
+    supported_protocols:
+    - completions
+    memory_class_gib: 4
+    min_vram_gib_per_replica: 4
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+  vicuna-7b-v1.3:
+    hf_model_id: lmsys/vicuna-7b-v1.3
+    served_model_name: lmsys/vicuna-7b-v1.3
+    logical_model_name: lmsys/vicuna-7b-v1.3
+    tokenizer_name: lmsys/vicuna-7b-v1.3
+    family: vicuna
+    supported_protocols:
+    - completions
+    notes:
+    - Use a no-chat-template style profile when reproducing legacy completion-style
+      prompting.
+    modalities:
+    - text
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16