siclaw 0.1.0 → 0.1.2

package/skills/core/volcano-node-resources/scripts/get-node-resources.sh

@@ -0,0 +1,281 @@
+#!/bin/bash
+# Query cluster node resources for Volcano scheduling.
+# This script performs read-only operations using kubectl.
+set -euo pipefail
+
+show_help() {
+  cat <<EOF
+Usage: $0 [options]
+
+Query cluster node resources to understand capacity and availability.
+Checks allocatable CPU, memory, GPU, and current usage.
+
+Options:
+  --node NODE       Query specific node only
+  --label LABEL     Filter nodes by label (e.g., gpu=true)
+  --show-usage      Show current resource usage (requires metrics-server)
+  --show-pods       Show pods running on each node
+  --format FORMAT   Output format: table (default), json, wide
+  -h, --help        Show this help message
+
+Examples:
+  $0                                  # All nodes
+  $0 --node worker-1                  # Specific node
+  $0 --label nvidia.com/gpu.present=true  # GPU nodes
+  $0 --show-usage --show-pods         # With usage and pods
+  $0 --format json                    # JSON output
+EOF
+  exit 0
+}
+
+# Parse arguments
+NODE=""
+LABEL=""
+SHOW_USAGE=false
+SHOW_PODS=false
+FORMAT="table"
+
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    -h|--help) show_help ;;
+    --node) NODE="$2"; shift 2 ;;
+    --label) LABEL="$2"; shift 2 ;;
+    --show-usage) SHOW_USAGE=true; shift ;;
+    --show-pods) SHOW_PODS=true; shift ;;
+    --format) FORMAT="$2"; shift 2 ;;
+    *) echo "Unknown option: $1. Use --help for usage." >&2; exit 1 ;;
+  esac
+done
+
+# Validate format
+if [[ "$FORMAT" != "table" && "$FORMAT" != "json" && "$FORMAT" != "wide" ]]; then
+  echo "Error: Invalid format '$FORMAT'. Use: table, json, or wide" >&2
+  exit 1
+fi
+
+echo "=== Volcano Node Resources ==="
+[[ -n "$NODE" ]] && echo "Node: $NODE"
+[[ -n "$LABEL" ]] && echo "Label filter: $LABEL"
+echo "Show usage: $SHOW_USAGE"
+echo "Show pods: $SHOW_PODS"
+echo "Format: $FORMAT"
+echo
+
+# Build kubectl get nodes command
+NODE_CMD="kubectl get nodes"
+[[ -n "$LABEL" ]] && NODE_CMD="$NODE_CMD -l $LABEL"
+[[ -n "$NODE" ]] && NODE_CMD="$NODE_CMD $NODE"
+
+# Check if nodes exist
+if ! $NODE_CMD -o name &>/dev/null; then
+  echo "Error: No nodes found matching criteria" >&2
+  exit 1
+fi
+
+# Function to get node resources
+get_node_resources() {
+  local node="$1"
+
+  # Get allocatable resources
+  local cpu_alloc mem_alloc gpu_alloc pods_alloc
+  cpu_alloc=$(kubectl get node "$node" -o jsonpath='{.status.allocatable.cpu}' 2>/dev/null || echo "N/A")
+  mem_alloc=$(kubectl get node "$node" -o jsonpath='{.status.allocatable.memory}' 2>/dev/null || echo "N/A")
+  gpu_alloc=$(kubectl get node "$node" -o jsonpath='{.status.allocatable.nvidia\.com/gpu}' 2>/dev/null || echo "0")
+  pods_alloc=$(kubectl get node "$node" -o jsonpath='{.status.allocatable.pods}' 2>/dev/null || echo "N/A")
+
+  # Get capacity
+  local cpu_cap mem_cap
+  cpu_cap=$(kubectl get node "$node" -o jsonpath='{.status.capacity.cpu}' 2>/dev/null || echo "N/A")
+  mem_cap=$(kubectl get node "$node" -o jsonpath='{.status.capacity.memory}' 2>/dev/null || echo "N/A")
+
+  # Get status and age
+  local status age
+  status=$(kubectl get node "$node" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || echo "Unknown")
+  # Calculate age in days (cross-platform: GNU date on Linux, BSD date on macOS)
+  age=$(kubectl get node "$node" -o jsonpath='{.metadata.creationTimestamp}' 2>/dev/null | {
+    IFS= read -r timestamp
+    if [[ -n "$timestamp" ]]; then
+      if date -d "$timestamp" +%s &>/dev/null 2>&1; then
+        # GNU date (Linux)
+        created=$(date -d "$timestamp" +%s 2>/dev/null)
+      else
+        # BSD date (macOS)
+        created=$(date -j -f "%Y-%m-%dT%H:%M:%SZ" "$timestamp" +%s 2>/dev/null)
+      fi
+      now=$(date +%s)
+      if [[ -n "$created" && -n "$now" ]]; then
+        echo $(( (now - created) / 86400 ))
+      else
+        echo "N/A"
+      fi
+    else
+      echo "N/A"
+    fi
+  })
+
+  # Get taints
+  local taints
+  taints=$(kubectl get node "$node" -o jsonpath='{.spec.taints[*].key}' 2>/dev/null || echo "")
+
+  # Get allocated resources (from describe)
+  local cpu_req mem_req
+  if describe_output=$(kubectl describe node "$node" 2>/dev/null); then
+    cpu_req=$(echo "$describe_output" | grep -A 5 "Allocated resources" | grep "cpu-requests" | awk '{print $2}' || echo "N/A")
+    mem_req=$(echo "$describe_output" | grep -A 5 "Allocated resources" | grep "memory-requests" | awk '{print $2}' || echo "N/A")
+  else
+    cpu_req="N/A"
+    mem_req="N/A"
+  fi
+
+  # Calculate available (rough estimate)
+  local cpu_avail="N/A"
+  local mem_avail="N/A"
+
+  # Try to calculate if we have numeric values
+  if [[ "$cpu_alloc" =~ ^[0-9]+$ && "$cpu_req" =~ ^[0-9]+m?$ ]]; then
+    # Convert millicores to cores if needed
+    local alloc_val req_val
+    alloc_val=$cpu_alloc
+    if [[ "$cpu_req" =~ m$ ]]; then
+      req_val=$(echo "${cpu_req%m}" | awk '{print $1/1000}')
+    else
+      req_val=$cpu_req
+    fi
+    cpu_avail=$(awk "BEGIN {printf \"%.0f\", $alloc_val - $req_val}")
+  fi
+
+  # Output based on format
+  case "$FORMAT" in
+    table)
+      echo "Node: $node"
+      echo "  Status: $status"
+      echo "  Age: ${age}d"
+      [[ -n "$taints" ]] && echo "  Taints: $taints"
+      echo "  Resources:"
+      echo "    CPU:        Allocatable=$cpu_alloc | Requested=$cpu_req | Available=$cpu_avail"
+      echo "    Memory:     Allocatable=$mem_alloc | Requested=$mem_req | Available=$mem_avail"
+      [[ "$gpu_alloc" != "0" ]] && echo "    GPU:        Allocatable=$gpu_alloc"
+      [[ "$SHOW_USAGE" == "true" ]] && echo "    Usage:      (see metrics below)"
+
+      if [[ "$SHOW_USAGE" == "true" ]]; then
+        echo
+        echo "  Resource Usage (requires metrics-server):"
+        if kubectl top node "$node" 2>/dev/null; then
+          : # success
+        else
+          echo "    (Metrics not available)"
+        fi
+      fi
+
+      if [[ "$SHOW_PODS" == "true" ]]; then
+        echo
+        echo "  Running Pods:"
+        kubectl get pods --all-namespaces --field-selector spec.nodeName="$node" -o custom-columns='NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,CPU_REQ:.spec.containers[*].resources.requests.cpu,MEM_REQ:.spec.containers[*].resources.requests.memory' 2>/dev/null | head -20 || echo "    (Failed to list pods)"
+      fi
+      echo
+      ;;
+
+    wide)
+      echo "$node $cpu_alloc $mem_alloc $gpu_alloc $status ${age}d"
+      ;;
+
+    json)
+      echo "  {"
+      echo "    \"name\": \"$node\","
+      echo "    \"status\": \"$status\","
+      echo "    \"age_days\": $age,"
+      [[ -n "$taints" ]] && echo "    \"taints\": \"$taints\","
+      echo "    \"allocatable\": {"
+      echo "      \"cpu\": \"$cpu_alloc\","
+      echo "      \"memory\": \"$mem_alloc\","
+      echo "      \"gpu\": \"$gpu_alloc\","
+      echo "      \"pods\": \"$pods_alloc\""
+      echo "    },"
+      echo "    \"requested\": {"
+      echo "      \"cpu\": \"$cpu_req\","
+      echo "      \"memory\": \"$mem_req\""
+      echo "    },"
+      echo "    \"available\": {"
+      echo "      \"cpu\": \"$cpu_avail\","
+      echo "      \"memory\": \"$mem_avail\""
+      echo "    }"
+      echo "  }"
+      ;;
+  esac
+}
+
+# Main logic
+case "$FORMAT" in
+  table|wide)
+    if [[ "$FORMAT" == "wide" ]]; then
+      echo "NAME CPU_ALLOC MEM_ALLOC GPU_ALLOC STATUS AGE"
+      echo "==== ========= ========= ========= ====== ===="
+    fi
+
+    if [[ -n "$NODE" ]]; then
+      get_node_resources "$NODE"
+    else
+      # Use process substitution instead of pipe to avoid subshell
+      while read -r n; do
+        [[ -n "$n" ]] && get_node_resources "$n"
+      done < <(kubectl get nodes ${LABEL:+-l $LABEL} -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n')
+    fi
+    ;;
+
+  json)
+    echo "{"
+    echo "  \"nodes\": ["
+
+    # Use process substitution to avoid subshell issue with 'first' variable
+    first=true
+    if [[ -n "$NODE" ]]; then
+      get_node_resources "$NODE"
+    else
+      while read -r n; do
+        if [[ -n "$n" ]]; then
+          [[ "$first" == "false" ]] && echo ","
+          get_node_resources "$n"
+          first=false
+        fi
+      done < <(kubectl get nodes ${LABEL:+-l $LABEL} -o jsonpath='{.items[*].metadata.name}' 2>/dev/null | tr ' ' '\n')
+    fi
+
+    echo
+    echo "  ]"
+    echo "}"
+    ;;
+esac
+
+# Summary for table format
+if [[ "$FORMAT" == "table" ]]; then
+  echo "=== Summary ==="
+
+  # Count nodes by status
+  total=$(kubectl get nodes ${LABEL:+-l $LABEL} 2>/dev/null | wc -l)
+  total=$((total - 1))  # Subtract header
+  ready=$(kubectl get nodes ${LABEL:+-l $LABEL} 2>/dev/null | grep -c " Ready " || echo "0")
+  not_ready=$((total - ready))
+
+  echo "Total Nodes: $total"
+  echo "  Ready: $ready"
+  echo "  NotReady: $not_ready"
+
+  # Check for GPU nodes
+  gpu_nodes=$(kubectl get nodes ${LABEL:+-l $LABEL} -o jsonpath='{.items[*].status.allocatable.nvidia\.com/gpu}' 2>/dev/null | tr ' ' '\n' | grep -v "^0$" | grep -v "^$" | wc -l)
+  if [[ "$gpu_nodes" -gt 0 ]]; then
+    echo "  GPU Nodes: $gpu_nodes"
+  fi
+
+  # Check metrics availability
+  if [[ "$SHOW_USAGE" == "true" ]]; then
+    echo
+    if kubectl top nodes &>/dev/null; then
+      echo "Metrics-server: Available"
+    else
+      echo "Metrics-server: Not Available (kubectl top nodes failed)"
+    fi
+  fi
+fi
+
+echo
+echo "=== Query Complete ==="

package/skills/core/volcano-queue-diagnose/SKILL.md

@@ -0,0 +1,294 @@
+---
+name: volcano-queue-diagnose
+description: >-
+  Diagnose Volcano Queue status and resource allocation.
+  Check queue weights, deserved resources, allocated resources,
+  and identify queue-related scheduling bottlenecks.
+---
+
+# Volcano Queue Diagnosis
+
+Diagnose Volcano Queue status, resource allocation, and scheduling bottlenecks. This skill helps understand how resources are distributed across queues and why workloads may be pending due to queue constraints.
+
+**Scope:** This skill is for **diagnosis only**. Once you identify the root cause, report it to the user and stop. Do NOT attempt to modify queue configurations or delete queues.
+
+**Not applicable to native ResourceQuota:** Volcano Queue and Kubernetes ResourceQuota are independent mechanisms. If the cluster does not use Volcano, use `quota-debug` instead. To check: `kubectl get queue 2>/dev/null` — if it returns an error or empty, Volcano is not installed.
+
+## Usage
+
+```bash
+bash skills/core/volcano-queue-diagnose/scripts/diagnose-queue.sh [options]
+```
+
+## Parameters
+
+| Parameter | Required | Description |
+|-----------|----------|-------------|
+| `--queue QUEUE` | no | Queue name to diagnose (default: all queues) |
+| `--show-pods` | no | Show pods associated with each queue |
+| `--verbose` | no | Show detailed resource breakdown |
+
+## Examples
+
+Diagnose all queues:
+```bash
+bash skills/core/volcano-queue-diagnose/scripts/diagnose-queue.sh
+```
+
+Diagnose specific queue:
+```bash
+bash skills/core/volcano-queue-diagnose/scripts/diagnose-queue.sh --queue training-queue
+```
+
+Show verbose output with pod information:
+```bash
+bash skills/core/volcano-queue-diagnose/scripts/diagnose-queue.sh --queue training-queue --show-pods --verbose
+```
+
+## Understanding Volcano Queues
+
+### Queue Concept
+
+In Volcano, a Queue is a cluster-level resource allocation unit. Jobs and PodGroups are submitted to queues, and the scheduler distributes resources among queues based on:
+
+1. **Weight** - Relative share of cluster resources (proportional: weight 10 vs weight 2 = 83% vs 17%)
+2. **Capability** - Maximum resources a queue can use (ceiling, not guarantee — actual allocation depends on cluster capacity and competition)
+3. **Parent** - Hierarchical queue relationships (if enabled)
+
+**Important:** A Queue is a **cluster-scoped** resource. PodGroups from **any namespace** can reference the same queue, so cross-namespace resource competition within a queue is expected.
+
+### Queue Status Fields
+
+| Field | Meaning |
+|-------|---------|
+| `state` | Queue state: Open, Closed, Closing |
+| `deserved` | Resources the queue should receive based on weight |
+| `allocated` | Resources currently allocated to jobs in this queue |
+| `used` | Resources actually used by running pods (≤ allocated) |
+| `pending` | Number of PodGroups waiting in the queue |
+| `running` | Number of running PodGroups |
+
+## Diagnostic Flow
+
+### Step 1: List All Queues
+
+Get an overview of all queues:
+
+```bash
+kubectl get queue
+```
+
+**Output columns:**
+- NAME: Queue name
+- WEIGHT: Queue weight (higher = more resources)
+- STATE: Open, Closed, or Closing
+- PARENT: Parent queue (for hierarchical queues)
+
+### Step 2: Check Queue Details
+
+Get detailed information about a specific queue:
+
+```bash
+kubectl get queue <queue-name> -o yaml
+kubectl describe queue <queue-name>
+```
+
+**Key sections to examine:**
+
+#### Spec (Configuration)
+```yaml
+spec:
+  weight: 10              # Relative weight (default: 1)
+  capability:             # Max resources allowed
+    cpu: "100"
+    memory: "200Gi"
+  reclaimable: true       # Allow resource reclamation
+```
+
+#### Status (Runtime State)
+```yaml
+status:
+  state: Open             # Open, Closed, or Closing
+  pending: 5              # PodGroups waiting
+  running: 10             # Running PodGroups
+  deserved:               # Resources this queue should get
+    cpu: "40"
+    memory: "80Gi"
+  allocated:              # Resources actually allocated
+    cpu: "35"
+    memory: "70Gi"
+```
+
+### Step 3: Check Queue Resource Utilization
+
+Calculate utilization ratios:
+
+```
+Allocation Ratio = allocated / deserved
+Utilization Ratio = used / allocated
+```
+
+**Interpretation:**
+- `allocated >= deserved`: Queue is at or over its fair share
+- `allocated < deserved`: Queue has room to grow
+- `used << allocated`: Jobs have reserved resources but not using them
+
+### Step 4: Identify PodGroups in Queue
+
+Find workloads associated with a queue:
+
+```bash
+# Find all PodGroups in a queue
+kubectl get podgroups --all-namespaces -o json | \
+  jq -r '.items[] | select(.spec.queue=="<queue-name>") | "\(.metadata.namespace)/\(.metadata.name)"'
+
+# Check pending PodGroups
+kubectl get podgroups --all-namespaces -o json | \
+  jq -r '.items[] | select(.spec.queue=="<queue-name>" and .status.phase=="Pending") | \
+  "\(.metadata.namespace)/\(.metadata.name): \(.status.phase)"'
+```
+
+### Step 5: Check Queue Events
+
+Look for queue-related events:
+
+```bash
+kubectl get events --all-namespaces --field-selector reason=FailedScheduling | grep -i queue
+```
+
+## Common Queue Issues
+
+### Issue 1: Queue Resource Exhaustion
+
+**Symptom:** `allocated >= deserved`, new PodGroups stay in Pending
+
+**Check:**
+```bash
+kubectl get queue <queue> -o jsonpath='{"
+  Deserved: "}{.status.deserved}{"
+  Allocated: "}{.status.allocated}{"
+  Ratio: "}{.status.allocated.cpu}{"/"}{.status.deserved.cpu}{"
+"}'
+```
+
+For GPU-specific checks (GPU is often the bottleneck):
+```bash
+kubectl get queue -o custom-columns="NAME:.metadata.name,GPU_CAP:.spec.capability['nvidia.com/gpu'],GPU_ALLOC:.status.allocated['nvidia.com/gpu']"
+```
+
+**Also cross-validate capability against actual cluster capacity** — a common misconfiguration is setting `spec.capability` higher than the cluster's physical resources:
+```bash
+kubectl get nodes -o custom-columns="NAME:.metadata.name,GPU:.status.allocatable['nvidia.com/gpu'],CPU:.status.allocatable.cpu,MEM:.status.allocatable.memory"
+```
+If the sum of all nodes' allocatable GPUs is less than the queue's `spec.capability`, the queue can never be fully utilized. When allocation reaches the cluster's physical limit, the queue appears to have remaining capacity but no more resources can actually be scheduled.
+
+**Solution:**
+- Increase queue weight (requires scheduler config change)
+- Increase queue capability (only if cluster has physical capacity)
+- Wait for other jobs to complete
+- Check if other queues are over-allocated (reclaim may help)
+
+### Issue 2: Queue is Closed
+
+**Symptom:** `status.state: Closed`, new PodGroups rejected
+
+**Check:**
+```bash
+kubectl get queue <queue> -o jsonpath='{.status.state}'
+```
+
+**Solution:**
+- Queue must be reopened by admin
+- Use a different queue
+
+### Issue 3: Weight Imbalance
+
+**Symptom:** One queue gets all resources, others starve
+
+**Check:**
+```bash
+kubectl get queue -o custom-columns='NAME:.metadata.name,WEIGHT:.spec.weight,STATE:.status.state,CPU_DESERVED:.status.deserved.cpu,CPU_ALLOC:.status.allocated.cpu,MEM_DESERVED:.status.deserved.memory,MEM_ALLOC:.status.allocated.memory'
+```
+
+**Analysis:** Volcano distributes resources proportionally by weight. For example:
+- Queue A (weight=10) + Queue B (weight=2): A gets 10/12 ≈ 83%, B gets 2/12 ≈ 17% of total cluster resources
+- If Queue B has many pending jobs but low deserved resources, its weight is too low relative to others
+
+**Solution:**
+- Adjust queue weights proportionally
+- Check if high-weight queues have capability limits preventing allocation
+
+### Issue 4: Resource Reclaim Not Working
+
+**Symptom:** Queue is over-allocated but reclaim is not triggered
+
+**Check:**
+```bash
+# Check reclaim is enabled in scheduler config
+kubectl get cm volcano-scheduler-configmap -n volcano-system -o yaml | grep reclaim
+```
+
+**Reclaim troubleshooting checklist (all must be true):**
+1. `reclaim` action must be in scheduler actions
+2. `proportion` plugin must be enabled
+3. Source queue must be under-utilized (allocated < deserved)
+4. Target queue must have over-allocated resources (allocated > deserved)
+5. Target queue must have `reclaimable: true`
+
+Check the reclaimable flag on the specific queue:
+```bash
+kubectl get queue <queue> -o jsonpath='{.spec.reclaimable}'
+```
+If `reclaimable` is `false` (or unset), the queue's resources **cannot be reclaimed** even if it's over-allocated.
+
+**Solution:**
+- Verify all 5 prerequisites above
+- Check scheduler logs for reclaim attempts: use `volcano-scheduler-logs --keyword reclaim`
+
+## Queue Hierarchy (Advanced)
+
+If using hierarchical queues:
+
+```bash
+# Check parent-child relationships
+kubectl get queue -o custom-columns='NAME:.metadata.name,PARENT:.spec.parent,WEIGHT:.spec.weight'
+```
+
+**Key points:**
+- Child queues share parent's deserved resources
+- Weight is relative to siblings, not absolute
+- Parent queue's deserved = sum of children's usage
+
+## Script Output Interpretation
+
+The diagnose-queue.sh script provides:
+
+1. **Queue Summary Table**
+   - Name, State, Weight
+   - Pending/Running counts
+   - Resource allocation summary
+
+2. **Resource Breakdown (with --verbose)**
+   - CPU: deserved, allocated, usage ratio
+   - Memory: deserved, allocated, usage ratio
+   - GPU: if available
+
+3. **Warning Flags**
+   - `[OVER]` - Queue allocated > deserved
+   - `[FULL]` - Queue at capacity
+   - `[CLOSED]` - Queue not accepting new jobs
+   - `[HIGH_PEND]` - Many pending PodGroups
+
+## Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `VOLCANO_NAMESPACE` | `default` | Default namespace for pod lookup |
+
+## See Also
+
+- `volcano-diagnose-pod` - Diagnose individual pod scheduling
+- `volcano-gang-scheduling` - Gang constraint issues
+- `volcano-resource-insufficient` - Resource shortage diagnosis
+- `volcano-scheduler-logs` - Check scheduler decisions
+- `quota-debug` - Native Kubernetes ResourceQuota/LimitRange diagnosis (non-Volcano)