infer-stack 0.6.0__py3-none-any.whl

infer_stack/templates/default-models.yaml

@@ -0,0 +1,674 @@
+models:
+  # Smallest Qwen3.5 — intended for fast smoke tests, local development,
+  # and high-concurrency utility tasks. Fits on a single 8 GiB GPU at FP16
+  # with room for activations and a modest KV cache.
+  qwen3.5-0.8b:
+    hf_model_id: Qwen/Qwen3.5-0.8B
+    served_model_name: qwen3.5-0.8b
+    family: qwen3.5
+    modalities: [text]
+    memory_class_gib: 4
+    min_vram_gib_per_replica: 4
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 32
+  qwen3.5-2b:
+    hf_model_id: Qwen/Qwen3.5-2B
+    served_model_name: qwen3.5-2b
+    family: qwen3.5
+    modalities: [text]
+    memory_class_gib: 8
+    min_vram_gib_per_replica: 8
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 32
+  qwen3.5-4b:
+    hf_model_id: Qwen/Qwen3.5-4B
+    served_model_name: qwen3.5-4b
+    family: qwen3.5
+    modalities: [text]
+    memory_class_gib: 12
+    min_vram_gib_per_replica: 12
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 24
+  qwen3.5-9b:
+    hf_model_id: Qwen/Qwen3.5-9B
+    served_model_name: qwen3.5-9b
+    family: qwen3.5
+    modalities: [text]
+    memory_class_gib: 20
+    min_vram_gib_per_replica: 20
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  qwen3.5-27b:
+    hf_model_id: Qwen/Qwen3.5-27B
+    served_model_name: qwen3.5-27b
+    family: qwen3.5
+    modalities: [text]
+    memory_class_gib: 48
+    min_vram_gib_per_replica: 48
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  qwen3.5-35b-a3b:
+    hf_model_id: Qwen/Qwen3.5-35B-A3B
+    served_model_name: qwen3.5-35b-a3b
+    family: qwen3.5
+    modalities: [text]
+    memory_class_gib: 48
+    min_vram_gib_per_replica: 48
+    preferred_gpu_count: 1
+    context_window: 32768
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+  qwen3.5-122b-a10b:
+    hf_model_id: Qwen/Qwen3.5-122B-A10B
+    served_model_name: qwen3.5-122b-a10b
+    family: qwen3.5
+    modalities: [text]
+    memory_class_gib: 80
+    min_vram_gib_per_replica: 80
+    preferred_gpu_count: 2
+    context_window: 131072
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 8
+
+  qwen3.5-122b-a10b-fp8:
+    hf_model_id: Qwen/Qwen3.5-122B-A10B-FP8
+    served_model_name: qwen3.5-122b-a10b-fp8
+    family: qwen3.5
+    modalities: [text]
+    memory_class_gib: 80
+    min_vram_gib_per_replica: 80
+    preferred_gpu_count: 2
+    context_window: 262144
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 8
+    tool_calling:
+      auto: true
+      parser: qwen3_xml
+
+  qwen3.6-35b-a3b:
+    hf_model_id: Qwen/Qwen3.6-35B-A3B
+    served_model_name: qwen3.6-35b-a3b
+    tokenizer_name: Qwen/Qwen3.6-35B-A3B
+    family: qwen3.6
+    modalities: [text]
+    memory_class_gib: 80
+    min_vram_gib_per_replica: 24
+    preferred_gpu_count: 2
+    context_window: 262144
+    supported_protocols: [chat, completions]
+    reasoning:
+      enabled: true
+      parser: qwen3
+      expose_to_openwebui: true
+    defaults:
+      max_model_len: 262144
+      gpu_memory_utilization: 0.95
+      enable_prefix_caching: false
+      max_num_batched_tokens: 8192
+      max_num_seqs: 8
+
+  gemma4-e2b:
+    hf_model_id: google/gemma-4-E2B-it
+    served_model_name: gemma4-e2b
+    family: gemma4
+    modalities: [text, image, audio]
+    memory_class_gib: 8
+    min_vram_gib_per_replica: 8
+    preferred_gpu_count: 1
+    context_window: 131072
+    thinking_history_policy: keep_final_only
+    notes:
+      - Historical turns should keep only final responses, not prior thinking content.
+    defaults:
+      max_model_len: 65536
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 16
+    tool_calling:
+      auto: true
+      parser: gemma4
+  gemma4-e4b:
+    hf_model_id: google/gemma-4-E4B-it
+    served_model_name: gemma4-e4b
+    family: gemma4
+    modalities: [text, image, audio]
+    memory_class_gib: 10
+    min_vram_gib_per_replica: 10
+    preferred_gpu_count: 1
+    context_window: 131072
+    thinking_history_policy: keep_final_only
+    notes:
+      - Historical turns should keep only final responses, not prior thinking content.
+    defaults:
+      max_model_len: 65536
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 16
+    tool_calling:
+      auto: true
+      parser: gemma4
+  gemma4-26b:
+    hf_model_id: google/gemma-4-26B-A4B-it
+    served_model_name: gemma4-26b
+    family: gemma4
+    modalities: [text, image]
+    memory_class_gib: 20
+    min_vram_gib_per_replica: 20
+    preferred_gpu_count: 1
+    context_window: 262144
+    thinking_history_policy: keep_final_only
+    notes:
+      - Historical turns should keep only final responses, not prior thinking content.
+    defaults:
+      max_model_len: 65536
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 12
+    tool_calling:
+      auto: true
+      parser: gemma4
+  gemma4-31b:
+    hf_model_id: google/gemma-4-31B-it
+    served_model_name: gemma4-31b
+    family: gemma4
+    modalities: [text, image]
+    memory_class_gib: 22
+    min_vram_gib_per_replica: 22
+    preferred_gpu_count: 1
+    context_window: 262144
+    thinking_history_policy: keep_final_only
+    notes:
+      - Historical turns should keep only final responses, not prior thinking content.
+    defaults:
+      max_model_len: 65536
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 12
+    tool_calling:
+      auto: true
+      parser: gemma4
+
+  # ── HELM evaluation models ────────────────────────────────────────────────
+  # served_model_name matches the HELM side tag so HELM can route correctly.
+
+  qwen2-72b-instruct:
+    hf_model_id: Qwen/Qwen2-72B-Instruct
+    served_model_name: qwen/qwen2-72b-instruct
+    logical_model_name: qwen/qwen2-72b-instruct
+    tokenizer_name: qwen/qwen2-72b-instruct
+    family: qwen2
+    modalities: [text]
+    memory_class_gib: 144
+    min_vram_gib_per_replica: 144
+    preferred_gpu_count: 2
+    context_window: 131072
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 8
+
+  # Smallest *real* instruct-tuned Qwen — usable for true end-to-end
+  # smoke tests that actually pull weights and exercise vLLM. ~1 GiB fp16.
+  qwen2.5-0.5b-instruct:
+    hf_model_id: Qwen/Qwen2.5-0.5B-Instruct
+    served_model_name: qwen/qwen2.5-0.5b-instruct
+    logical_model_name: qwen/qwen2.5-0.5b-instruct
+    tokenizer_name: Qwen/Qwen2.5-0.5B-Instruct
+    family: qwen2.5
+    modalities: [text]
+    memory_class_gib: 2
+    min_vram_gib_per_replica: 2
+    preferred_gpu_count: 1
+    context_window: 32768
+    defaults:
+      max_model_len: 8192
+      gpu_memory_utilization: 0.85
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 16
+
+  qwen2.5-7b-instruct:
+    hf_model_id: Qwen/Qwen2.5-7B-Instruct
+    served_model_name: qwen/qwen2.5-7b-instruct-turbo
+    logical_model_name: qwen/qwen2.5-7b-instruct-turbo
+    tokenizer_name: qwen/qwen2.5-7b-instruct-turbo
+    family: qwen2.5
+    modalities: [text]
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 131072
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+
+  qwen2.5-72b-instruct:
+    hf_model_id: Qwen/Qwen2.5-72B-Instruct
+    served_model_name: qwen/qwen2.5-72b-instruct-turbo
+    logical_model_name: qwen/qwen2.5-72b-instruct-turbo
+    tokenizer_name: qwen/qwen2.5-72b-instruct-turbo
+    family: qwen2.5
+    modalities: [text]
+    memory_class_gib: 144
+    min_vram_gib_per_replica: 144
+    preferred_gpu_count: 2
+    context_window: 131072
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 8
+
+  gpt-oss-20b:
+    hf_model_id: openai/gpt-oss-20b
+    served_model_name: openai/gpt-oss-20b
+    logical_model_name: openai/gpt-oss-20b
+    tokenizer_name: openai/o200k_harmony
+    family: gpt-oss
+    modalities: [text]
+    memory_class_gib: 40
+    min_vram_gib_per_replica: 40
+    preferred_gpu_count: 1
+    context_window: 131072
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+
+  llama-3.1-8b-instruct:
+    hf_model_id: meta-llama/Meta-Llama-3.1-8B-Instruct
+    served_model_name: meta/llama-3.1-8b-instruct-turbo
+    family: llama3.1
+    modalities: [text]
+    memory_class_gib: 18
+    min_vram_gib_per_replica: 18
+    preferred_gpu_count: 1
+    context_window: 131072
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+
+  llama-3-8b-instruct:
+    hf_model_id: meta-llama/Meta-Llama-3-8B-Instruct
+    served_model_name: meta/llama-3-8b-chat
+    family: llama3
+    modalities: [text]
+    memory_class_gib: 18
+    min_vram_gib_per_replica: 18
+    preferred_gpu_count: 1
+    context_window: 8192
+    defaults:
+      max_model_len: 8192
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+
+  llama-2-7b:
+    hf_model_id: meta-llama/Llama-2-7b-hf
+    served_model_name: meta/llama-2-7b
+    family: llama2
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 4096
+    defaults:
+      max_model_len: 4096
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 16
+
+  llama-2-13b:
+    hf_model_id: meta-llama/Llama-2-13b-hf
+    served_model_name: meta/llama-2-13b
+    family: llama2
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 28
+    min_vram_gib_per_replica: 28
+    preferred_gpu_count: 1
+    context_window: 4096
+    defaults:
+      max_model_len: 4096
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 16
+
+  # LLaMA-2-70B at fp16 needs ~140 GB; the default tp=2 layout fits on
+  # two 96 GB GPUs with headroom for KV cache. Public HELM ran fp16,
+  # so the audit recipe matches that — no quantization in this profile.
+  llama-2-70b:
+    hf_model_id: meta-llama/Llama-2-70b-hf
+    served_model_name: meta/llama-2-70b
+    family: llama2
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 140
+    min_vram_gib_per_replica: 70
+    preferred_gpu_count: 2
+    context_window: 4096
+    defaults:
+      max_model_len: 4096
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 4096
+      max_num_seqs: 8
+
+  mistral-7b-instruct-v0.3:
+    hf_model_id: mistralai/Mistral-7B-Instruct-v0.3
+    served_model_name: mistralai/mistral-7b-instruct-v0.3
+    family: mistral
+    modalities: [text]
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 32768
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+
+  mistral-7b-v0.1:
+    hf_model_id: mistralai/Mistral-7B-v0.1
+    served_model_name: mistralai/mistral-7b-v0.1
+    family: mistral
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 32768
+    defaults:
+      max_model_len: 32768
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: true
+      max_num_batched_tokens: 8192
+      max_num_seqs: 16
+
+  # Falcon uses ALiBi attention: prefix caching must be disabled.
+  falcon-7b:
+    hf_model_id: tiiuae/falcon-7b
+    served_model_name: tiiuae/falcon-7b
+    family: falcon
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+
+  falcon-7b-instruct:
+    hf_model_id: tiiuae/falcon-7b-instruct
+    served_model_name: tiiuae/falcon-7b-instruct
+    family: falcon
+    modalities: [text]
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+
+  # GPT-2 base, 124M params (~250 MB fp16). Completions-only — no chat
+  # template. Useful as a near-zero-cost plumbing smoke test: download is
+  # fast, no HF_TOKEN required, and the request path covers the whole stack.
+  gpt2:
+    hf_model_id: openai-community/gpt2
+    served_model_name: gpt2
+    logical_model_name: gpt2
+    tokenizer_name: openai-community/gpt2
+    family: gpt2
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 1
+    min_vram_gib_per_replica: 1
+    preferred_gpu_count: 1
+    context_window: 1024
+    defaults:
+      max_model_len: 1024
+      gpu_memory_utilization: 0.3
+      enable_prefix_caching: true
+      max_num_batched_tokens: 1024
+      max_num_seqs: 16
+
+  # Smallest practical instruct-tuned model on HuggingFace. ~270 MB fp16.
+  # Ideal for CI smoke tests that need real chat-template output.
+  smollm2-135m-instruct:
+    hf_model_id: HuggingFaceTB/SmolLM2-135M-Instruct
+    served_model_name: huggingfacetb/smollm2-135m-instruct
+    logical_model_name: huggingfacetb/smollm2-135m-instruct
+    tokenizer_name: HuggingFaceTB/SmolLM2-135M-Instruct
+    family: smollm2
+    modalities: [text]
+    memory_class_gib: 1
+    min_vram_gib_per_replica: 1
+    preferred_gpu_count: 1
+    context_window: 8192
+    defaults:
+      max_model_len: 4096
+      gpu_memory_utilization: 0.6
+      enable_prefix_caching: true
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+
+  # Pythia uses ALiBi attention: prefix caching must be disabled.
+  pythia-160m:
+    hf_model_id: EleutherAI/pythia-160m
+    served_model_name: eleutherai/pythia-160m
+    logical_model_name: eleutherai/pythia-160m
+    tokenizer_name: EleutherAI/pythia-160m
+    family: pythia
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 1
+    min_vram_gib_per_replica: 1
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.6
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+
+  pythia-410m:
+    hf_model_id: EleutherAI/pythia-410m
+    served_model_name: eleutherai/pythia-410m
+    logical_model_name: eleutherai/pythia-410m
+    tokenizer_name: EleutherAI/pythia-410m
+    family: pythia
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 2
+    min_vram_gib_per_replica: 2
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.7
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+
+  pythia-6.9b:
+    hf_model_id: EleutherAI/pythia-6.9b
+    served_model_name: eleutherai/pythia-6.9b
+    logical_model_name: eleutherai/pythia-6.9b
+    tokenizer_name: eleutherai/pythia-6.9b
+    family: pythia
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+
+  pythia-2.8b-v0:
+    hf_model_id: EleutherAI/pythia-2.8b-v0
+    served_model_name: eleutherai/pythia-2.8b-v0
+    logical_model_name: eleutherai/pythia-2.8b-v0
+    tokenizer_name: eleutherai/pythia-2.8b-v0
+    family: pythia
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 8
+    min_vram_gib_per_replica: 8
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+
+  pythia-1b-v0:
+    hf_model_id: EleutherAI/pythia-1b-v0
+    served_model_name: eleutherai/pythia-1b-v0
+    logical_model_name: eleutherai/pythia-1b-v0
+    tokenizer_name: eleutherai/pythia-1b-v0
+    family: pythia
+    modalities: [text]
+    supported_protocols: [completions]
+    memory_class_gib: 4
+    min_vram_gib_per_replica: 4
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16
+
+  vicuna-7b-v1.3:
+    hf_model_id: lmsys/vicuna-7b-v1.3
+    served_model_name: lmsys/vicuna-7b-v1.3
+    logical_model_name: lmsys/vicuna-7b-v1.3
+    tokenizer_name: lmsys/vicuna-7b-v1.3
+    family: vicuna
+    supported_protocols: [completions]
+    notes:
+      - Use a no-chat-template style profile when reproducing legacy completion-style prompting.
+    modalities: [text]
+    memory_class_gib: 16
+    min_vram_gib_per_replica: 16
+    preferred_gpu_count: 1
+    context_window: 2048
+    defaults:
+      max_model_len: 2048
+      gpu_memory_utilization: 0.9
+      enable_prefix_caching: false
+      max_num_batched_tokens: 2048
+      max_num_seqs: 16

infer_stack/templates/default-ollama-models.yaml

@@ -0,0 +1,31 @@
+ollama_models:
+  qwen3.5-2b:
+    tag: qwen3.5:2b
+    served_model_name: qwen3.5-2b
+    defaults:
+      num_ctx: 4096
+      temperature: 0.2
+  qwen3.5-4b:
+    tag: qwen3.5:4b
+    served_model_name: qwen3.5-4b
+    defaults:
+      num_ctx: 4096
+      temperature: 0.2
+  qwen3.5-9b-q4:
+    tag: qwen3.5:9b-q4_K_M
+    served_model_name: qwen3.5-9b-q4
+    defaults:
+      num_ctx: 4096
+      temperature: 0.2
+  gemma4-e2b:
+    tag: gemma4:e2b
+    served_model_name: gemma4-e2b
+    defaults:
+      num_ctx: 4096
+      temperature: 0.2
+  smollm2-135m:
+    tag: smollm2:135m
+    served_model_name: smollm2-135m
+    defaults:
+      num_ctx: 4096
+      temperature: 0.2