siclaw 0.1.1 → 0.1.2

package/skills/core/volcano-queue-diagnose/scripts/diagnose-queue.sh

@@ -0,0 +1,283 @@
+#!/bin/bash
+# Diagnose Volcano Queue status and resource allocation.
+# This script performs read-only operations using kubectl.
+set -euo pipefail
+
+if ! command -v jq &>/dev/null; then
+  echo "ERROR: jq is required but not installed. Install it with: apt-get install jq / brew install jq" >&2
+  exit 1
+fi
+
+show_help() {
+  cat <<EOF
+Usage: $0 [options]
+
+Diagnose Volcano Queue status, resource allocation, and scheduling bottlenecks.
+Checks queue weights, deserved resources, allocated resources, and state.
+
+Options:
+  --queue QUEUE     Queue name to diagnose (default: all queues)
+  --show-pods       Show PodGroups associated with each queue
+  --verbose         Show detailed resource breakdown
+  -h, --help        Show this help message
+
+Examples:
+  $0                              # Diagnose all queues
+  $0 --queue training-queue       # Diagnose specific queue
+  $0 --queue training-queue --verbose --show-pods
+EOF
+  exit 0
+}
+
+# Parse arguments
+QUEUE=""
+SHOW_PODS=false
+VERBOSE=false
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -h|--help) show_help ;;
+    --queue) QUEUE="$2"; shift 2 ;;
+    --show-pods) SHOW_PODS=true; shift ;;
+    --verbose) VERBOSE=true; shift ;;
+    *) echo "Unknown option: $1. Use --help for usage." >&2; exit 1 ;;
+  esac
+done
+
+echo "=== Volcano Queue Diagnosis ==="
+[[ -n "$QUEUE" ]] && echo "Queue: $QUEUE"
+echo
+
+# Function to diagnose a single queue
+diagnose_queue() {
+  local q="$1"
+
+  echo "Queue: $q"
+  echo "=================="
+
+  # Get queue details
+  local weight state pending running deserved allocated
+  weight=$(kubectl get queue "$q" -o jsonpath='{.spec.weight}' 2>/dev/null || echo "N/A")
+  state=$(kubectl get queue "$q" -o jsonpath='{.status.state}' 2>/dev/null || echo "Unknown")
+  pending=$(kubectl get queue "$q" -o jsonpath='{.status.pending}' 2>/dev/null || echo "0")
+  running=$(kubectl get queue "$q" -o jsonpath='{.status.running}' 2>/dev/null || echo "0")
+
+  echo "  Weight: $weight"
+  echo "  State: $state"
+  echo "  Pending PodGroups: $pending"
+  echo "  Running PodGroups: $running"
+
+  # State warnings
+  if [[ "$state" == "Closed" ]]; then
+    echo "  ⚠️  WARNING: Queue is CLOSED - new jobs will be rejected"
+  elif [[ "$state" == "Closing" ]]; then
+    echo "  ⚠️  WARNING: Queue is CLOSING - will not accept new jobs soon"
+  fi
+
+  # Get resource info
+  local deserved_cpu deserved_mem allocated_cpu allocated_mem
+  deserved_cpu=$(kubectl get queue "$q" -o jsonpath='{.status.deserved.cpu}' 2>/dev/null || echo "")
+  deserved_mem=$(kubectl get queue "$q" -o jsonpath='{.status.deserved.memory}' 2>/dev/null || echo "")
+  allocated_cpu=$(kubectl get queue "$q" -o jsonpath='{.status.allocated.cpu}' 2>/dev/null || echo "")
+  allocated_mem=$(kubectl get queue "$q" -o jsonpath='{.status.allocated.memory}' 2>/dev/null || echo "")
+
+  # Handle empty values
+  [[ -z "$deserved_cpu" ]] && deserved_cpu="0"
+  [[ -z "$deserved_mem" ]] && deserved_mem="0"
+  [[ -z "$allocated_cpu" ]] && allocated_cpu="0"
+  [[ -z "$allocated_mem" ]] && allocated_mem="0"
+
+  echo
+  echo "  Resources:"
+  echo "    CPU:"
+  echo "      Deserved:   $deserved_cpu"
+  echo "      Allocated:  $allocated_cpu"
+
+  # Calculate CPU ratio if possible
+  if [[ "$deserved_cpu" =~ ^[0-9]+\.?[0-9]*$ && "$allocated_cpu" =~ ^[0-9]+\.?[0-9]*$ ]]; then
+    if awk "BEGIN {exit !($deserved_cpu > 0)}" 2>/dev/null; then
+      local cpu_ratio
+      cpu_ratio=$(awk "BEGIN {printf \"%.1f\", $allocated_cpu * 100 / $deserved_cpu}" 2>/dev/null || echo "N/A")
+      echo "      Ratio:      ${cpu_ratio}%"
+
+      if awk "BEGIN {exit !($allocated_cpu > $deserved_cpu)}" 2>/dev/null; then
+        echo "      ⚠️  OVER-ALLOCATED: Queue using more than deserved"
+      elif [[ "$cpu_ratio" != "N/A" ]] && awk "BEGIN {exit !($cpu_ratio >= 90)}" 2>/dev/null; then
+        echo "      ⚠️  NEAR CAPACITY: ${cpu_ratio}% of deserved resources used"
+      fi
+    fi
+  fi
+
+  echo "    Memory:"
+  echo "      Deserved:   $deserved_mem"
+  echo "      Allocated:  $allocated_mem"
+
+  # Check capability if set
+  local cap_cpu cap_mem
+  cap_cpu=$(kubectl get queue "$q" -o jsonpath='{.spec.capability.cpu}' 2>/dev/null || echo "")
+  cap_mem=$(kubectl get queue "$q" -o jsonpath='{.spec.capability.memory}' 2>/dev/null || echo "")
+
+  if [[ -n "$cap_cpu" || -n "$cap_mem" ]]; then
+    echo "    Capability (max allowed):"
+    [[ -n "$cap_cpu" ]] && echo "      CPU: $cap_cpu"
+    [[ -n "$cap_mem" ]] && echo "      Memory: $cap_mem"
+  fi
+
+  # Check reclaimable
+  local reclaimable
+  reclaimable=$(kubectl get queue "$q" -o jsonpath='{.spec.reclaimable}' 2>/dev/null || echo "true")
+  [[ "$reclaimable" != "false" ]] && reclaimable="true"
+  echo "    Reclaimable: $reclaimable"
+  [[ "$reclaimable" == "false" ]] && echo "      ℹ️  Resources cannot be reclaimed from this queue"
+
+  # Show PodGroups if requested
+  if [[ "$SHOW_PODS" == "true" ]]; then
+    echo
+    echo "  PodGroups in this Queue:"
+    local pgs
+    pgs=$(kubectl get podgroups --all-namespaces -o json 2>/dev/null | \
+      jq -r --arg q "$q" '.items[] | select(.spec.queue==$q) | "    \(.metadata.namespace)/\(.metadata.name): \(.status.phase // \"Unknown\")"' 2>/dev/null || echo "")
+
+    if [[ -n "$pgs" ]]; then
+      echo "$pgs"
+    else
+      echo "    No PodGroups found"
+    fi
+
+    # Show pending count specifically
+    local pending_pgs
+    pending_pgs=$(kubectl get podgroups --all-namespaces -o json 2>/dev/null | \
+      jq -r --arg q "$q" '.items[] | select(.spec.queue==$q and .status.phase=="Pending") | "\(.metadata.namespace)/\(.metadata.name)"' 2>/dev/null || echo "")
+
+    if [[ -n "$pending_pgs" ]]; then
+      echo
+      echo "  ⚠️  Pending PodGroups:"
+      echo "$pending_pgs" | while read -r pg; do
+        echo "    - $pg"
+      done
+    fi
+  fi
+
+  # Verbose output
+  if [[ "$VERBOSE" == "true" ]]; then
+    echo
+    echo "  Raw Queue YAML:"
+    kubectl get queue "$q" -o yaml 2>/dev/null | sed 's/^/    /'
+  fi
+
+  echo
+}
+
+# Main logic
+if [[ -n "$QUEUE" ]]; then
+  # Diagnose specific queue
+  if ! kubectl get queue "$QUEUE" &>/dev/null; then
+    echo "Error: Queue '$QUEUE' not found" >&2
+    exit 1
+  fi
+  diagnose_queue "$QUEUE"
+else
+  # Diagnose all queues
+  echo "[1] Listing all queues"
+  echo "---------------------"
+  kubectl get queue -o custom-columns='NAME:.metadata.name,STATE:.status.state,WEIGHT:.spec.weight,PENDING:.status.pending,RUNNING:.status.running' 2>/dev/null || {
+    echo "Error: Failed to list queues" >&2
+    exit 1
+  }
+  echo
+
+  echo "[2] Resource Allocation Summary"
+  echo "--------------------------------"
+  # Print table header
+  printf "%-20s %-8s %-10s %-12s %-15s %-12s\n" "QUEUE" "STATE" "WEIGHT" "CPU_RATIO" "MEM_ALLOC" "PODS(P/R)"
+  printf "%-20s %-8s %-10s %-12s %-15s %-12s\n" "--------------------" "--------" "----------" "------------" "---------------" "-----------"
+  
+  # Helper: convert K8s CPU value (e.g. "500m", "2", "1.5") to millicores
+  cpu_to_milli() {
+    local v="$1"
+    if [[ "$v" =~ ^([0-9]+)m$ ]]; then
+      echo "${BASH_REMATCH[1]}"
+    elif [[ "$v" =~ ^[0-9]+\.?[0-9]*$ ]]; then
+      awk "BEGIN {printf \"%.0f\", $v * 1000}" 2>/dev/null || echo "0"
+    else
+      echo "0"
+    fi
+  }
+
+  # Get all queue names and print resource summary
+  kubectl get queue -o json 2>/dev/null | jq -r '.items[] |
+    [.metadata.name,
+     (.status.state // "Unknown"),
+     (.spec.weight // 1),
+     (.status.deserved.cpu // "0"),
+     (.status.allocated.cpu // "0"),
+     (.status.allocated.memory // "N/A"),
+     (.status.pending // 0),
+     (.status.running // 0)] | @tsv' 2>/dev/null | \
+  while IFS=$'\t' read -r name state weight deserved_cpu_raw alloc_cpu_raw mem_alloc pending running; do
+    ratio=0
+    deserved_m=$(cpu_to_milli "$deserved_cpu_raw")
+    alloc_m=$(cpu_to_milli "$alloc_cpu_raw")
+    if [[ "$deserved_m" -gt 0 ]]; then
+      ratio=$((alloc_m * 100 / deserved_m))
+    fi
+
+    status_indicator=""
+    if [[ "$state" == "Closed" ]]; then
+      status_indicator="🚫"
+    elif [[ "$state" == "Closing" ]]; then
+      status_indicator="⚠️"
+    elif [[ "$ratio" -ge 90 ]]; then
+      status_indicator="🔴"
+    elif [[ "$ratio" -ge 75 ]]; then
+      status_indicator="🟡"
+    fi
+
+    printf "%-20s %-8s %-10s %-12s %-15s %-12s %s\n" \
+      "$name" "$state" "${weight:-1}" "${ratio}%" "${mem_alloc:-N/A}" "${pending:-0}/${running:-0}" "$status_indicator"
+  done
+  echo
+  echo "Legend: 🚫=Closed ⚠️=Closing 🔴=>=90% 🟡=>=75%"
+  echo
+
+  echo "[3] Detailed Queue Analysis"
+  echo "--------------------------"
+  # Get all queue names
+  kubectl get queue -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n' | while read -r q; do
+    [[ -n "$q" ]] && diagnose_queue "$q"
+  done
+fi
+
+# Summary
+echo "=== Diagnosis Summary ==="
+
+# Count queues by state
+total=0
+open_count=0
+closed_count=0
+closing_count=0
+
+while read -r q; do
+  [[ -z "$q" ]] && continue
+  total=$((total + 1))
+  state=$(kubectl get queue "$q" -o jsonpath='{.status.state}' 2>/dev/null || echo "Unknown")
+  [[ "$state" == "Open" ]] && open_count=$((open_count + 1))
+  [[ "$state" == "Closed" ]] && closed_count=$((closed_count + 1))
+  [[ "$state" == "Closing" ]] && closing_count=$((closing_count + 1))
+done < <(kubectl get queue -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n')
+
+echo "Total Queues: $total"
+echo "  Open: $open_count"
+echo "  Closed: $closed_count"
+echo "  Closing: $closing_count"
+
+# Find queues with high pending
+high_pending=$(kubectl get queue -o json 2>/dev/null | jq -r '.items[] | select(.status.pending > 5) | "\(.metadata.name) (\(.status.pending) pending)"' 2>/dev/null || echo "")
+if [[ -n "$high_pending" ]]; then
+  echo
+  echo "⚠️  Queues with high pending (>5):"
+  echo "$high_pending" | sed 's/^/  - /'
+fi
+
+echo
+echo "=== Diagnosis Complete ==="

package/skills/core/volcano-resource-insufficient/SKILL.md

@@ -0,0 +1,315 @@
+---
+name: volcano-resource-insufficient
+description: >-
+  Resource shortage diagnostic guide for Volcano.
+  Use when seeing Insufficient cpu/memory events, OOMKilled pods,
+  or nodes with zero allocatable resources.
+---
+
+# Resource Insufficiency Diagnosis
+
+This guide helps diagnose resource shortage issues in Volcano-scheduled workloads. Resource insufficiency is one of the most common causes of scheduling failures.
+
+**Scope:** This skill is for **diagnosis only**. Once you identify the root cause, report it to the user and stop. Do NOT attempt to modify resource quotas or delete workloads.
+
+## When to Use This Guide
+
+Use this skill when:
+- Events show `Insufficient cpu` or `Insufficient memory`
+- Pods are stuck in `Pending` with resource-related events
+- Nodes show zero allocatable resources
+- Pods are being `OOMKilled` (Out of Memory)
+- `FailedScheduling` events mention resource constraints
+
+## Types of Resource Issues
+
+### 1. Cluster-Wide Resource Exhaustion
+The entire cluster lacks sufficient resources to meet the workload demands.
+
+### 2. Resource Fragmentation
+Total resources exist but are distributed across too many nodes to satisfy specific scheduling constraints (like Gang scheduling).
+
+### 3. Per-Node Resource Shortage
+Individual nodes lack enough resources, even though the cluster as a whole has capacity.
+
+### 4. Queue Resource Limits
+The Queue has reached its deserved resource limit, preventing new pods from being scheduled.
+
+## Diagnostic Steps
+
+### Step 1: Identify Resource Shortage Type
+
+Check the specific error message in events:
+
+```bash
+kubectl get events -n <namespace> --field-selector involvedObject.name=<pod-name> --sort-by='.lastTimestamp'
+```
+
+**Common error patterns:**
+
+| Error Message | Resource Type | Scope |
+|--------------|---------------|-------|
+| `Insufficient cpu` | CPU | Node-level |
+| `Insufficient memory` | Memory | Node-level |
+| `Insufficient nvidia.com/gpu` | GPU | Node-level |
+| `0/N nodes are available` | General | Cluster-level |
+| `exceeded quota` | Queue-level | Queue limit |
+
+### Step 2: Check Pod Resource Requests
+
+Determine how much resources the pod is requesting:
+
+```bash
+kubectl get pod <pod-name> -n <namespace> -o jsonpath='{.spec.containers[*].resources.requests}'
+```
+
+For detailed breakdown:
+```bash
+kubectl get pod <pod-name> -n <namespace> -o yaml | grep -A 10 "resources:"
+```
+
+**Key fields:**
+- `resources.requests.cpu` - CPU cores requested (e.g., "100m" = 0.1 core, "2" = 2 cores)
+- `resources.requests.memory` - Memory requested (e.g., "1Gi", "512Mi")
+- `resources.requests.nvidia.com/gpu` - GPUs requested
+- `resources.limits` - Maximum allowed (may be different from requests)
+
+### Step 3: Check Node Allocatable Resources
+
+View total allocatable resources per node:
+
+```bash
+kubectl get nodes -o custom-columns='NAME:.metadata.name,CPU:.status.allocatable.cpu,MEM:.status.allocatable.memory,GPU:.status.allocatable.nvidia\.com/gpu,PODS:.status.allocatable.pods'
+```
+
+For detailed node information:
+```bash
+kubectl describe node <node-name>
+```
+
+**Key concepts:**
+- `allocatable` = Total capacity - System reserved - Kubelet reserved
+- `capacity` = Total hardware capacity
+- The difference is reserved for system/Kubernetes daemons
+
+### Step 4: Check Current Resource Usage
+
+If metrics-server is available:
+
+```bash
+kubectl top nodes
+```
+
+For per-node pod usage:
+```bash
+kubectl top pods --all-namespaces --sort-by=cpu | head -20
+kubectl top pods --all-namespaces --sort-by=memory | head -20
+```
+
+**Note:** If metrics-server is not available, you can still see resource allocation (requests) but not actual usage.
+
+### Step 5: Calculate Resource Availability
+
+For each node, calculate available resources:
+
+```
+Available CPU = allocatable.cpu - sum(all pod requests on node)
+Available Memory = allocatable.memory - sum(all pod requests on node)
+```
+
+Quick check with:
+```bash
+kubectl describe node <node-name> | grep -A 20 "Allocated resources"
+```
+
+**Look for:**
+- `cpu-requests` vs `cpu-capacity`
+- `memory-requests` vs `memory-capacity`
+- Percentage of allocation (high % = resource pressure)
+
+### Step 6: Check for Resource Fragmentation
+
+For Gang scheduling or affinity constraints, fragmentation is critical:
+
+```bash
+# Count nodes that can fit a single pod
+NODE_CPU_REQ="4"
+NODE_MEM_REQ="8Gi"
+
+kubectl get nodes -o json | jq -r '
+  .items[] |
+  select(.status.allocatable.cpu | tonumber >= '"$NODE_CPU_REQ"') |
+  select(.status.allocatable.memory | ascii_downcase | gsub("[gimk]"; "") | tonumber >= 8) |
+  .metadata.name'
+```
+
+**Fragmentation indicators:**
+- Many nodes with small amounts of free resources
+- No single node can satisfy the pod's resource needs
+- Total cluster resources sufficient but poorly distributed
+
+## Common Scenarios
+
+### Scenario 1: Pod Requests Exceed Any Single Node
+
+**Symptom:** `Insufficient cpu` or `Insufficient memory` on all nodes
+
+**Diagnosis:**
+```bash
+# Check pod request
+kubectl get pod <pod> -o jsonpath='{.spec.containers[0].resources.requests.cpu}'
+# Output: 32
+
+# Check largest node's allocatable
+kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{"\t"}{.status.allocatable.cpu}{"\n"}{end}' | sort -k2 -n | tail -1
+# Output: node-1    16
+```
+
+**Analysis:** Pod requests 32 CPUs, but largest node only has 16 allocatable.
+
+**Solution:**
+- Reduce pod resource requests (if actual usage is lower)
+- Add larger nodes to cluster
+- Use node pool with bigger instances
+
+### Scenario 2: Cluster at Capacity
+
+**Symptom:** Most nodes show high allocation percentage
+
+**Diagnosis:**
+```bash
+kubectl describe node <node-name> | grep "Allocated resources"
+# cpu-requests:  14900m (93%)
+# memory-requests:  55000Mi (85%)
+```
+
+**Analysis:** Nodes are 85-93% allocated, leaving little room for new pods.
+
+**Solution:**
+- Scale cluster (add more nodes)
+- Review and optimize resource requests (may be over-provisioned)
+- Consider cluster autoscaler for dynamic scaling
+
+### Scenario 3: Resource Fragmentation
+
+**Symptom:** Gang scheduling fails despite total resources being sufficient
+
+**Diagnosis:**
+```bash
+# Total cluster CPU
+kubectl get nodes -o jsonpath='{range .items[*]}{.status.allocatable.cpu}{"\n"}{end}' | awk '{sum+=$1} END {print sum}'
+# Output: 64
+
+# Available nodes for 4-CPU pods
+kubectl get nodes -o custom-columns='NAME:.metadata.name,CPU:.status.allocatable.cpu' | awk '$2 >= 4 {count++} END {print count " nodes can fit the pod"}'
+# Output: 2 nodes can fit the pod
+
+# But we need 8 pods for Gang
+# 2 < 8, so Gang fails
+```
+
+**Analysis:** Total cluster has 64 CPUs, but only 2 nodes have 4+ CPUs. Gang needs 8 pods.
+
+**Solution:**
+- Enable `binpack` plugin to concentrate pods
+- Defragment by draining and rebalancing nodes
+- Use larger nodes to reduce fragmentation
+
+### Scenario 4: Queue Resource Exhaustion
+
+**Symptom:** Events mention queue limits, PodGroup stays in Pending
+
+**Diagnosis:**
+```bash
+# Check queue status
+kubectl get queue <queue-name> -o yaml
+```
+
+**Look for:**
+- `status.allocated` >= `status.deserved`
+- `state` is `Open` but no capacity available
+
+**Analysis:** Queue has used all its deserved resources.
+
+**Solution:**
+- Increase queue weight or capability
+- Wait for other jobs to complete
+- Use `volcano-queue-diagnose` for detailed analysis
+
+### Scenario 5: GPU Resource Shortage
+
+**Symptom:** `Insufficient nvidia.com/gpu` in events
+
+**Diagnosis:**
+```bash
+# Check GPU allocatable
+kubectl get nodes -o custom-columns='NAME:.metadata.name,GPU:.status.allocatable.nvidia\.com/gpu'
+
+# Check GPU usage (if metrics available)
+kubectl top nodes --show-capacity 2>/dev/null || echo "GPU metrics not available"
+```
+
+**Analysis:** GPU resources are fully allocated or fragmented across nodes.
+
+**Solution:**
+- Verify GPU device plugin is running
+- Check if GPUs are properly allocatable on nodes
+- Consider GPU sharing if workload supports it
+
+## Resource Calculation Examples
+
+### Example 1: Calculate Total Cluster Capacity
+
+```bash
+kubectl get nodes -o json | jq -r '
+  .items |
+  map(.status.allocatable) |
+  reduce .[] as $item ({}; 
+    . + {cpu: ((.cpu // 0 | tonumber) + ($item.cpu | tonumber)),
+         memory: ((.memory // 0) + ($item.memory | tonumber))})'
+```
+
+### Example 2: Find Pods with High Resource Requests
+
+```bash
+kubectl get pods --all-namespaces -o json | jq -r '
+  .items[] |
+  select(.spec.containers[].resources.requests.cpu | tonumber > 4) |
+  "\(.metadata.namespace)/\(.metadata.name): \(.spec.containers[].resources.requests)"'
+```
+
+### Example 3: Check Resource Utilization vs Request
+
+```bash
+# High-level view
+kubectl get nodes -o custom-columns='NAME:.metadata.name,CPU_ALLOC:.status.allocatable.cpu,MEM_ALLOC:.status.allocatable.memory'
+```
+
+## Prevention and Best Practices
+
+1. **Right-size resource requests**
+   - Set requests based on actual usage, not maximum possible
+   - Use Vertical Pod Autoscaler (VPA) for recommendations
+
+2. **Use cluster autoscaler**
+   - Automatically scale nodes based on pending pod demands
+   - Configure appropriate node pools for different workloads
+
+3. **Enable binpack plugin**
+   - Reduces fragmentation by concentrating pods
+   - Better for batch workloads
+
+4. **Monitor resource quotas**
+   - Set up alerts for queue resource exhaustion
+   - Use `volcano-queue-diagnose` proactively
+
+5. **Regular capacity planning**
+   - Track resource growth trends
+   - Plan cluster expansion before hitting capacity
+
+## See Also
+
+- `volcano-diagnose-pod` - General Pod scheduling diagnosis
+- `volcano-gang-scheduling` - Gang scheduling constraint issues
+- `volcano-queue-diagnose` - Queue resource analysis
+- `volcano-node-resources` - Node resource querying