terradev-cli 5.2.1__tar.gz → 5.3.1__tar.gz

{terradev_cli-5.2.1 → terradev_cli-5.3.1}/Deployment Package/BNF_GRAMMAR.md

@@ -1,5 +1,5 @@
 # Terradev CLI BNF Grammar
-# Complete syntax specification for all commands and options — v3.6.2
+# Complete syntax specification for all commands and options — v5.3.0
 
 <terradev-cli> ::= <global-options> <command>
 
@@ -90,6 +90,7 @@
     | <triggers-commands>
     | <environments-commands>
     | <lineage-commands>
+    | <agent-commands>
 
 
 # ════════════════════════════════════════════════════════════════════
@@ -1053,6 +1054,53 @@
 <lineage-direction>   ::= up | down | both
 
 
+# ════════════════════════════════════════════════════════════════════
+# AGENT FLEET COMMAND GROUP  (v6.0.0)
+# ════════════════════════════════════════════════════════════════════
+
+<agent-commands> ::= agent <agent-subcommand>
+
+<agent-subcommand> ::=
+    | plan      <agent-plan-options>
+    | deploy    <agent-deploy-options>
+    | status    --fleet-id <string>
+    | scale     <agent-scale-options>
+    | cost      --fleet-id <string>
+    | list
+    | teardown  --fleet-id <string> [--force]
+
+<agent-plan-options> ::=
+    --agents <integer>
+    [--model <string>]
+    [--reasoning <agent-reasoning-mode>]
+    [--planner-gpu <gpu-type>]
+    [--worker-gpu <gpu-type>]
+    [--planner-count <integer>]
+    [--worker-count <integer>]
+    [--output <path>]
+
+<agent-deploy-options> ::=
+    --agents <integer>
+    [--model <string>]
+    [--reasoning <agent-reasoning-mode>]
+    [--planner-gpu <gpu-type>]
+    [--worker-gpu <gpu-type>]
+    [--planner-count <integer>]
+    [--worker-count <integer>]
+    [--providers <provider>]...
+    [--max-price <float>]
+    [--dry-run]
+
+<agent-scale-options> ::=
+    --fleet-id <string>
+    --tier <agent-tier>
+    --count <integer>
+    [--providers <provider>]...
+
+<agent-reasoning-mode> ::= instant | deep | mixed
+<agent-tier>           ::= reasoning | decode | cpu_tools
+
+
 # ════════════════════════════════════════════════════════════════════
 # LOCAL GPU POOL COMMAND GROUP
 # ════════════════════════════════════════════════════════════════════

{terradev_cli-5.2.1 → terradev_cli-5.3.1}/Deployment Package/architecture.md

@@ -162,9 +162,11 @@ terradev infer-deploy --model ./llama-2-7b
              └── vLLM Router (P/D disaggregation)
 ```
 
-**Disaggregated Prefill/Decode (advanced):**
+**Disaggregated Prefill/Decode — Transport-Agnostic Layer:**
 
 ```
+core/pd_transport.py — KVTransport abstraction
+
 Client request
        │
        ▼
@@ -172,14 +174,124 @@ vLLM Router (sticky routing by prefix hash)
        │
        ├──▶ Prefill pool (H100, compute-bound)
        │    Processes input prompt
-       │    KV cache → NIXL zero-copy RDMA transfer
-       │
+       │    KV cache block ──► TransportSelector.select(config)
+       │                             │
+       │              ┌──────────────┼──────────────┐
+       │              ▼              ▼              ▼
+       │         NIXLNVLink      NIXL/IB        TCP fallback
+       │         600 GB/s       200 GB/s        10 GB/s
+       │         (NVLink 4.0)   (HDR IB)        (always avail)
+       │              │              │              │
+       │              └──────────────┴──────────────┘
+       │                             │
        └──▶ Decode pool (H100, memory-bound)
-            Generates tokens
+            Receives KV block, generates tokens
             Subsequent requests with same prefix
             route here to reuse KV cache
 ```
 
+**Transport selection priority** (probed at provision time, ~2s):
+
+| Priority | Transport | Bandwidth | Latency | Requirement |
+|----------|-----------|-----------|---------|-------------|
+| 1 | NIXL/NVLink | 600 GB/s | 0.05ms | NVLink 4.0 fabric |
+| 2 | NIXL/InfiniBand | 200–400 GB/s | 0.2ms | HDR/NDR IB |
+| 3 | CXL 3.0 | 200 GB/s | 0.0001ms | **PLANNED** |
+| 4 | RoCE RDMA | 25 GB/s | 1ms | 200GbE NIC |
+| 5 | TCP fallback | 10 GB/s | 0.5ms | Always available |
+
+**NIXL → CXL Migration Path (planned):**
+
+```
+CURRENT (2025):  NIXL protocol — zero-copy RDMA via NVLink or InfiniBand.
+                 Prefill GPU serializes KV → RDMA transfer → Decode GPU.
+                 Production-ready in vLLM ≥0.6.x, SGLang ≥0.4.x.
+
+PHASE 1 (2026):  NIXL + CXL co-existence.
+                 When CXL 3.0 fabric detected (Intel Clearwater Forest /
+                 AMD Venice), route KV through CXL pool. Fall back to NIXL.
+                 Expected bandwidth: ~200 GB/s. Latency: ~100ns.
+
+PHASE 2 (2026–2027): CXL-primary.
+                 KV cache allocated in CXL shared DRAM pool, not GPU HBM.
+                 Prefill writes directly to CXL; decode reads in-place.
+                 No serialization, no transfer — pointer handoff only.
+                 GPU HBM used only for model weights + active compute.
+                 VRAM sizing changes: replace GPU_VRAM_GB with CXL_POOL_GB
+                 in AgentTopologyPlanner._compute_kv_budget().
+
+PHASE 3 (2027+): CXL fabric switch (Astera Labs Atlas, Microchip Igloo).
+                 N prefill + M decode nodes share one CXL memory pool.
+                 Multi-agent KV sharing becomes a memory management problem:
+                 shared prefix maps to one physical address, accessed by all.
+```
+
+---
+
+### Multi-Agent KV Cache Sharing (v5.3.0)
+
+**The problem:** Each agent independently stores its KV cache. For N agents
+with 70% shared context (system prompt + task), the shared portion is stored
+N times — pure waste that scales linearly with fleet size.
+
+**The math** (from `core/kv_sharing.py`):
+
+```
+20 agents × Llama-70B × 32K context (fp16):
+  KV per agent:      32K × 80 layers × 512 bytes/tok/layer ÷ 2 = 0.655 GB
+  Naive (N copies):  20 × 0.655 = 13.1 GB per GPU slot
+
+  With broadcast sharing (70% shared = 22.4K shared, 9.6K unique):
+    Shared KV (stored once):  22.4K × 0.0205 GB/K = 0.46 GB
+    Per-agent unique KV:       9.6K × 0.0205 GB/K = 0.197 GB each
+    Total fleet KV:           0.46 + 20 × 0.197   = 4.4 GB
+    Saving: 13.1 - 4.4 = 8.7 GB (66% reduction)
+    → 3× more agents per GPU
+    → 3× fewer GPUs needed
+    → ~$14/hr savings on a 20-agent H100 fleet
+
+Eviction cost without sharing (H100 SXM, 32K context):
+  Re-prefill time: 32,768 / 30,000 tokens/sec ≈ 1.1s per eviction
+  With 20 agents on 6 GPUs (6 fit per GPU = 36 slots): no evictions
+  With 20 agents on 2 GPUs (6 fit per GPU = 12 slots): ~8 overflow agents
+  → 8 × 3,600 / 30s turns × 1.1s = 1,056s/hr wasted ≈ 29% throughput lost
+```
+
+**CLI:**
+```
+terradev provision -g H100 --agents 20 --context 32k --model-name llama-70b
+terradev provision -g H100 --agents 20 --context 32k --sharing-topology broadcast --dry-run
+```
+
+**Output** (what no other CLI computes today):
+```
+KV Sharing Plan — 20 agents × llama-70b @ 32K ctx
+Topology: broadcast  |  Shared prefix: 22K tokens
+
+  VRAM without sharing: 13.1 GB (6 agents/GPU → 4 GPUs needed)
+  VRAM with sharing:     4.4 GB (18 agents/GPU → 2 GPUs needed)
+  Saving: 8.7 GB (66% reduction)
+
+  Cost/hr without sharing: $9.96
+  Cost/hr with sharing:    $4.98  (saves $4.98/hr = $119/day)
+  ✓ With sharing: re-prefill overhead negligible (<1%)
+
+  TIER             INSTANCES         GPU   TP   CONC  CONTEXT   $/HR
+  ────────────────────────────────────────────────────────────────────
+  reasoning                2     H100_SXM    1      4     32K  $ 4.98
+  decode                   2  A100_SXM_80    2      2     32K  $ 5.96
+  cpu_tools                3          CPU    1     10     n/a  $ 1.80
+  ────────────────────────────────────────────────────────────────────
+  TOTAL                                                        $12.74/hr
+```
+
+**New files:**
+- `core/pd_transport.py` — `KVTransport` ABC, `NIXLNVLinkTransport`, `NIXLIBTransport`,
+  `CXLTransport` (stub, Phase 1), `RDMARoCETransport`, `TCPFallbackTransport`,
+  `TransportSelector`, `transfer_time_ms()`
+- `core/kv_sharing.py` — `MultiAgentVRAMPlanner`, `KVSharingPlan`, `AgentKVBudget`,
+  `EvictionCostModel`, `SharingTopology` enum
+
 ### Kubernetes Layer
 
 ```
@@ -335,14 +447,84 @@ environment_manager
 
 ---
 
+## Agentic Fleet Provisioning (v5.3.0)
+
+Purpose-built heterogeneous GPU fleet management for multi-agent LLM workloads.
+Research basis: arXiv:2605.26297 "Agentic AI Workload Characteristics" (2026).
+
+**Key empirical findings driving the design:**
+- Decode dominates: 91–98% of LLM execution time is decode, not prefill
+- KV cache hit rates: 84.6–99.5% when context stays resident — eviction = recompute
+- Context footprint: avg 37K–80K tokens, P95 tail up to 166K tokens
+- Tool execution: 2–29% of total runtime (retrieval workloads: 25–29%)
+
+```
+terradev agent deploy --agents 16 --model meta-llama/Llama-3.1-70B-Instruct
+       │
+       ├── AgentTopologyPlanner.infer_from_agent_count(n=16, model)
+       │     ├── Parse model size (70B)
+       │     ├── Select reasoning GPU: H100 SXM (KV cache preservation)
+       │     ├── Select decode GPU: A100 SXM 80GB × 2 (TP=2, bandwidth-optimised)
+       │     ├── CPU tools: 48-vCPU instances (Bash/WebFetch/file ops)
+       │     └── AgentFleetSpec  [fleet_id, tiers, networking, autoscaling, cost]
+       │
+       ├── AgenticProvisioner.provision_fleet(spec)
+       │     ├── Wave 0: quote_reasoning ‖ quote_decode ‖ quote_cpu
+       │     ├── Wave 1: provision_reasoning ‖ provision_decode ‖ provision_cpu
+       │     │           (via ParallelProvisioner across configured clouds)
+       │     ├── Wave 2: configure_networking (placement group, VPC peering)
+       │     ├── Wave 3: deploy_reasoning_inference ‖ deploy_decode_inference
+       │     │           (vLLM: --enable-prefix-caching, --kv-connector=offloading)
+       │     └── Wave 4: register_fleet → ~/.terradev/fleets/<fleet_id>.json
+       │
+       └── InferenceRouter (KV prefix-aware routing)
+             ├── PrefixCacheIndex: route by prompt prefix hash
+             ├── Sticky routing: X-Agent-Id header → same pod
+             └── Preserves KV residency across agent turns
+```
+
+**Three-tier hardware model:**
+
+| Tier | Hardware | Optimised For | Autoscale Signal |
+|------|----------|---------------|------------------|
+| reasoning | H100 SXM 80GB | KV cache preservation, long context | TTFT p95 > 2000ms |
+| decode | A100 SXM 80GB × 2 (TP=2) | Memory bandwidth, token throughput | Decode queue depth > 6 |
+| cpu_tools | 48-vCPU (no GPU) | Bash, WebFetch, Read/Edit/Grep | Tool latency p95 > 400ms |
+
+**Why NOT GPU utilization for autoscaling:**
+Agentic workloads are inherently bursty — GPU utilization oscillates between 0% (tool
+execution) and 100% (decode burst) within a single agent turn. TTFT and queue depth
+are stable signals that reflect actual user-visible latency.
+
+**CLI commands:**
+```
+terradev agent plan     --agents 16 --model ...     # size fleet, show cost
+terradev agent deploy   --agents 16 --model ...     # provision all tiers
+terradev agent status   --fleet-id ag_xxx           # KV hit rate, TTFT, queue
+terradev agent scale    --fleet-id ag_xxx --tier decode --count 8
+terradev agent cost     --fleet-id ag_xxx           # per-tier spend breakdown
+terradev agent list                                 # all known fleets
+terradev agent teardown --fleet-id ag_xxx           # destroy + remove state
+```
+
+**MCP tools (6 new in v6.0.0):**
+`agent_topology_plan`, `agent_fleet_provision`, `agent_fleet_status`,
+`agent_fleet_scale`, `agent_fleet_cost`, `agent_fleet_teardown`
+
+---
+
 ## File Map
 
 ```
 terradev_cli/
-  cli.py                        Main CLI (~11,600 lines, Click)
+  cli.py                        Main CLI (~14,400 lines, Click)
   cli_karpenter.py              Karpenter subcommand group
   cli_hf_spaces.py              HuggingFace Spaces subcommand group
   core/
+    agentic_topology.py         AgentFleetSpec, AgentTopologyPlanner (v5.3.0)
+    agentic_provisioner.py      AgenticProvisioner, fleet state mgmt (v5.3.0)
+    pd_transport.py             Transport-agnostic P/D KV layer (v5.3.0)
+    kv_sharing.py               Multi-agent KV sharing VRAM planner (v5.3.0)
     training_orchestrator.py    DeepSpeed/torchrun/FlashOptim
     provision_orchestrator.py   Parallel provision + NUMA
     evaluation_orchestrator.py  Model/endpoint evaluation
@@ -350,6 +532,10 @@ terradev_cli/
     cost_tracker.py             Spend tracking
     price_intelligence.py       Spot market analytics + ML
     trace_viewer.py             Phoenix span tree renderer
+    dag_executor.py             Wave-parallel DAG execution (Rust-backed)
+    parallel_provisioner.py     Multi-cloud parallel provisioning
+    warm_pool_manager.py        Pre-warming strategies for bursty workloads
+    inference_router.py         KV prefix-aware routing + auto-failover
   ml_services/
     __init__.py
     vllm_service.py             vLLM config + LoRA + sleep
@@ -364,10 +550,13 @@ terradev_cli/
     kubernetes_service.py
     kubernetes_enhanced.py      GPU operator, MIG, monitoring
 clusters/
+  agentic-template/             Heterogeneous agent fleet (v6.0.0)
+    helm/values-agentic.yaml    Three-tier Helm values
+    k8s/fleet-manifests.yaml    Deployments, HPAs, NetworkPolicies, Karpenter
   moe-template/                 MoE K8s + Helm (all opts auto-applied)
   rag-template/                 Qdrant + Phoenix + Guardrails
   glm-5/                        GLM-5 production templates
-rust/                           NVML bindings, DAG orchestrator
+rust/                           NVML bindings, DAG orchestrator, MCP codec
 terradev-mcp/
-  terradev_mcp.py               218-tool MCP server (~6,700 lines)
+  terradev_mcp.py               304-tool MCP server (~8,700 lines)
 ```

{terradev_cli-5.2.1/terradev_cli.egg-info → terradev_cli-5.3.1}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: terradev-cli
-Version: 5.2.1
+Version: 5.3.1
 Summary: An imperative command-line-interface for AI workload orchestration.
 Author-email: Terradev Team <team@terradev.com>
 Maintainer-email: Terradev Team <team@terradev.com>
@@ -93,7 +93,7 @@ Requires-Dist: redis>=5.0.0; extra == "enterprise"
 Requires-Dist: PyJWT>=2.8.0; extra == "enterprise"
 Dynamic: license-file
 
-# Terradev CLI v5.2.1
+# Terradev CLI v5.3.1
 
 **An imperative command-line-interface for AI workload orchestration.**
 
@@ -105,6 +105,20 @@ pypi.org/project/terradev-cli/
 
 Terradev is a cross-cloud compute-provisioning CLI that compresses + stages datasets, provisions optimal instances + nodes, and deploys **3-5x faster** than sequential provisioning.
 
+**NOTES ON 5.3.0**
+
+Added **transport-agnostic Prefill/Decode (P/D) disaggregation layer** and **multi-agent KV cache sharing planner**:
+
+- **Transport-agnostic P/D layer** (`core/pd_transport.py`): KV cache transfer is abstracted across NIXL/NVLink (600 GB/s), NIXL/InfiniBand (200–400 GB/s), CXL 3.0 (planned migration), RoCE RDMA, and TCP fallback. The `TransportSelector` probes at provision time and selects the best available transport. Documented NIXL→CXL migration path (Phase 1 co-existence → Phase 2 CXL-primary → Phase 3 fabric switch).
+
+- **Multi-agent KV sharing planner** (`core/kv_sharing.py`): Computes fleet VRAM requirements with KV cache sharing topologies (broadcast/star/chain/none). For 20 agents with 70% shared context (broadcast), VRAM reduces by 66% → 3× more agents per GPU, 3× fewer GPUs needed, ~$14/hr savings on H100 fleet. Includes `EvictionCostModel` to quantify re-prefill penalty when VRAM is under-provisioned.
+
+- **CLI integration**: `terradev provision` now supports `--agents`, `--context`, `--sharing-topology`, `--dtype` flags. When `--agents` is passed, the KV planner outputs the heterogeneous fleet spec with cost savings:
+  ```bash
+  terradev provision -g H100 --agents 20 --context 32k --model-name llama-70b
+  terradev provision -g H100 --agents 20 --context 32k --sharing-topology broadcast --dry-run
+  ```
+
 **NOTES ON 5.2.1**
 
 Added two new BYOAPI providers: **Yotta Labs (Shakti Cloud)** and **E2E Networks** — India's leading GPU clouds. Yotta Labs uses a pod-based compute model (similar to RunPod), and E2E Networks is a traditional VM-style hyperscaler that is NSE-listed and MeitY empanelled. Both are BYOAPI: your key, stored locally, never touches a Terradev server.

{terradev_cli-5.2.1/terradev_cli → terradev_cli-5.3.1}/README.md

@@ -1,4 +1,4 @@
-# Terradev CLI v5.2.1
+# Terradev CLI v5.3.1
 
 **An imperative command-line-interface for AI workload orchestration.**
 
@@ -10,6 +10,20 @@ pypi.org/project/terradev-cli/
 
 Terradev is a cross-cloud compute-provisioning CLI that compresses + stages datasets, provisions optimal instances + nodes, and deploys **3-5x faster** than sequential provisioning.
 
+**NOTES ON 5.3.0**
+
+Added **transport-agnostic Prefill/Decode (P/D) disaggregation layer** and **multi-agent KV cache sharing planner**:
+
+- **Transport-agnostic P/D layer** (`core/pd_transport.py`): KV cache transfer is abstracted across NIXL/NVLink (600 GB/s), NIXL/InfiniBand (200–400 GB/s), CXL 3.0 (planned migration), RoCE RDMA, and TCP fallback. The `TransportSelector` probes at provision time and selects the best available transport. Documented NIXL→CXL migration path (Phase 1 co-existence → Phase 2 CXL-primary → Phase 3 fabric switch).
+
+- **Multi-agent KV sharing planner** (`core/kv_sharing.py`): Computes fleet VRAM requirements with KV cache sharing topologies (broadcast/star/chain/none). For 20 agents with 70% shared context (broadcast), VRAM reduces by 66% → 3× more agents per GPU, 3× fewer GPUs needed, ~$14/hr savings on H100 fleet. Includes `EvictionCostModel` to quantify re-prefill penalty when VRAM is under-provisioned.
+
+- **CLI integration**: `terradev provision` now supports `--agents`, `--context`, `--sharing-topology`, `--dtype` flags. When `--agents` is passed, the KV planner outputs the heterogeneous fleet spec with cost savings:
+  ```bash
+  terradev provision -g H100 --agents 20 --context 32k --model-name llama-70b
+  terradev provision -g H100 --agents 20 --context 32k --sharing-topology broadcast --dry-run
+  ```
+
 **NOTES ON 5.2.1**
 
 Added two new BYOAPI providers: **Yotta Labs (Shakti Cloud)** and **E2E Networks** — India's leading GPU clouds. Yotta Labs uses a pod-based compute model (similar to RunPod), and E2E Networks is a traditional VM-style hyperscaler that is NSE-listed and MeitY empanelled. Both are BYOAPI: your key, stored locally, never touches a Terradev server.

terradev_cli-5.3.1/clusters/agentic-template/helm/values-agentic.yaml

@@ -0,0 +1,222 @@
+# Agentic Fleet Helm Values
+#
+# Three-tier heterogeneous GPU fleet for multi-agent LLM workloads.
+# Grounded in arXiv:2605.26297 — decode dominates (91-98%), KV cache hit
+# rates 84-99%, context footprint 37K-166K tokens.
+#
+# Usage:
+#   helm install agent-fleet ./charts/agentic-fleet -f values-agentic.yaml
+#
+# Tiers:
+#   reasoning  — H100 SXM, KV cache preservation, long context
+#   decode     — A100 80GB, memory-bandwidth-optimised token streaming
+#   cpu_tools  — High-vCPU, Bash/WebFetch/file-op tool execution
+
+global:
+  model: "meta-llama/Llama-3.1-70B-Instruct"
+  fleet_id: ""            # set by terradev agent deploy
+  n_agents: 16
+  image:
+    repository: vllm/vllm-openai
+    tag: latest
+    pullPolicy: IfNotPresent
+
+# ── Reasoning tier ────────────────────────────────────────────────────────────
+# KV preservation is the primary objective.
+# Research: context averages 37K-80K tokens; we provision for 120K (P95 tail).
+# Failed agents accumulate MORE context — provision for the tail, not the mean.
+reasoning:
+  enabled: true
+  replicaCount: 2
+  gpu:
+    type: "H100_SXM"
+    countPerPod: 1
+  resources:
+    limits:
+      nvidia.com/gpu: "1"
+      memory: "200Gi"
+      cpu: "16"
+    requests:
+      nvidia.com/gpu: "1"
+      memory: "180Gi"
+      cpu: "12"
+  vllm:
+    tensorParallelSize: 1
+    maxModelLen: 120000      # 120K context budget (arXiv P95 tail)
+    gpuMemoryUtilization: 0.90
+    enablePrefixCaching: true  # CRITICAL: 84-99% KV hit rate requires this
+    kvConnector: "offloading"  # CPU offload when under pressure (avoid eviction)
+    speculativeConfig:
+      method: "mtp"
+    enableSleepMode: true
+    maxNumSeqs: 8            # concurrency per instance
+    serveOnPort: 8000
+  autoscaling:
+    enabled: true
+    minReplicas: 1
+    maxReplicas: 8
+    metrics:
+      # Scale on TTFT degradation, NOT GPU utilization (bursty workload)
+      - type: External
+        external:
+          metric:
+            name: vllm_request_ttft_seconds_p95
+          target:
+            type: Value
+            value: "2"    # 2 seconds TTFT threshold
+    scaleInCooldown: 300     # 5 min — protect KV cache from eviction during scale-in
+    scaleOutCooldown: 30
+  nodeSelector:
+    terradev.ai/tier: reasoning
+    cloud.google.com/gke-accelerator: nvidia-h100-sxm
+
+# ── Decode tier ───────────────────────────────────────────────────────────────
+# Memory-bandwidth-optimised for token streaming.
+# A100 SXM 80GB: 2 TB/s HBM2e — sweet spot for 70B decode at batch=8.
+# Research: decode = 91-98% of LLM time → optimise for throughput here.
+decode:
+  enabled: true
+  replicaCount: 4           # default for 16 agents; adjusted by agentic_provisioner.py
+  gpu:
+    type: "A100_SXM_80"
+    countPerPod: 2           # TP=2 for 70B model (140GB weights need 2 × 80GB)
+  resources:
+    limits:
+      nvidia.com/gpu: "2"
+      memory: "400Gi"
+      cpu: "16"
+    requests:
+      nvidia.com/gpu: "2"
+      memory: "350Gi"
+      cpu: "12"
+  vllm:
+    tensorParallelSize: 2
+    maxModelLen: 120000
+    gpuMemoryUtilization: 0.90
+    enablePrefixCaching: true
+    kvConnector: "offloading"
+    speculativeConfig:
+      method: "mtp"
+    enableSleepMode: true
+    maxNumSeqs: 16           # higher concurrency — decode tier handles bulk of traffic
+    serveOnPort: 8001
+  autoscaling:
+    enabled: true
+    minReplicas: 1
+    maxReplicas: 32
+    metrics:
+      # Scale on decode queue depth (backlog of pending tool-call continuations)
+      - type: External
+        external:
+          metric:
+            name: vllm_num_requests_waiting
+          target:
+            type: Value
+            value: "6"    # >6 waiting requests → scale out
+    scaleInCooldown: 90
+    scaleOutCooldown: 15
+  nodeSelector:
+    terradev.ai/tier: decode
+    cloud.google.com/gke-accelerator: nvidia-a100-80gb
+
+# ── CPU tools tier ────────────────────────────────────────────────────────────
+# For Bash, Read/Edit/Grep, WebFetch/WebSearch tool execution.
+# Research: tools = 2-29% of runtime. GAIA (retrieval) hits 25-29%.
+# High vCPU count (48) enables parallel tool call fan-out across agent steps.
+cpuTools:
+  enabled: true
+  replicaCount: 2
+  resources:
+    limits:
+      memory: "96Gi"
+      cpu: "48"
+    requests:
+      memory: "64Gi"
+      cpu: "32"
+  image:
+    repository: terradev/agent-tool-runner
+    tag: latest
+  env:
+    MAX_CONCURRENT_TOOLS: "64"
+    BASH_TIMEOUT_S: "30"
+    WEBFETCH_TIMEOUT_S: "15"
+    TOOL_QUEUE_DEPTH_WARN: "8"
+  autoscaling:
+    enabled: true
+    minReplicas: 1
+    maxReplicas: 16
+    metrics:
+      - type: Resource
+        resource:
+          name: cpu
+          target:
+            type: Utilization
+            averageUtilization: 70
+      # Also scale on p95 tool latency
+      - type: External
+        external:
+          metric:
+            name: agent_tool_execution_latency_p95_ms
+          target:
+            type: Value
+            value: "400"   # 400ms p95 threshold
+    scaleInCooldown: 30
+    scaleOutCooldown: 10
+  nodeSelector:
+    terradev.ai/tier: cpu-tools
+
+# ── Inference router (optional) ───────────────────────────────────────────────
+# Routes agent requests: KV prefix-aware routing to the instance most likely
+# to have the context cached (avoids redundant prefill recompute).
+# Uses existing InferenceRouter.PrefixCacheIndex from inference_router.py.
+router:
+  enabled: true
+  replicaCount: 1
+  image:
+    repository: terradev/inference-router
+    tag: latest
+  config:
+    # Route reasoning requests to reasoning tier
+    reasoningEndpoint: "http://reasoning-svc:8000"
+    # Route decode requests to decode tier
+    decodeEndpoint: "http://decode-svc:8001"
+    # KV prefix cache index: route by prompt prefix hash
+    prefixCacheEnabled: true
+    prefixCacheMaxEntries: 10000
+    prefixTokens: 64
+    # Sticky routing: same agent_id → same pod (preserves KV residency)
+    stickyRouting: true
+    stickyHeader: "X-Agent-Id"
+
+# ── Monitoring ────────────────────────────────────────────────────────────────
+# Prometheus metrics for autoscaling signal derivation.
+# Key metrics: vllm_kv_cache_usage_perc, vllm_num_requests_waiting,
+#   vllm_request_ttft_seconds (histogram), agent_tool_execution_latency_ms
+monitoring:
+  enabled: true
+  prometheus:
+    enabled: true
+    serviceMonitor: true
+    scrapeInterval: 10s
+  grafana:
+    enabled: true
+    dashboards:
+      agentFleet: true        # KV hit rate, TTFT, queue depth, tier cost
+  dcgm:
+    enabled: true             # GPU memory utilization, bandwidth saturation
+  alerts:
+    kvHitRateBelow: 0.80      # arXiv threshold for healthy caching
+    ttftP95AboveMs: 2000      # reasoning tier TTFT threshold
+    decodeQueueDepthAbove: 8  # decode backlog alert
+    toolLatencyP95AboveMs: 500
+
+# ── Networking ────────────────────────────────────────────────────────────────
+networking:
+  placementGroup: cluster    # co-locate all tiers in same AZ for low latency
+  interTierLatencyTargetMs: 2
+  bandwidthGbps: 25
+  rdmaEnabled: false          # enable for 70B+ models requiring KV RDMA transfer
+  networkPolicies:
+    enabled: true
+    # Only reasoning and decode tiers can talk to each other
+    # CPU tools tier can only reach external APIs and result consumers