shipwright-cli 3.1.0 → 3.3.0

package/scripts/lib/daemon-poll-health.sh

@@ -0,0 +1,299 @@
+# daemon-poll-health.sh — Health checks and degradation detection for daemon-poll.sh
+# Source from daemon-poll.sh. Requires state, helpers.
+[[ -n "${_DAEMON_POLL_HEALTH_LOADED:-}" ]] && return 0
+_DAEMON_POLL_HEALTH_LOADED=1
+
+# Defaults for variables normally set by sw-daemon.sh (safe under set -u).
+DAEMON_DIR="${DAEMON_DIR:-${HOME}/.shipwright}"
+STATE_FILE="${STATE_FILE:-${DAEMON_DIR}/daemon-state.json}"
+PAUSE_FLAG="${PAUSE_FLAG:-${DAEMON_DIR}/daemon-pause.flag}"
+SHUTDOWN_FLAG="${SHUTDOWN_FLAG:-${DAEMON_DIR}/daemon.shutdown}"
+EVENTS_FILE="${EVENTS_FILE:-${DAEMON_DIR}/events.jsonl}"
+SCRIPT_DIR="${SCRIPT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+NO_GITHUB="${NO_GITHUB:-false}"
+POLL_INTERVAL="${POLL_INTERVAL:-60}"
+MAX_PARALLEL="${MAX_PARALLEL:-4}"
+WATCH_LABEL="${WATCH_LABEL:-shipwright}"
+WATCH_MODE="${WATCH_MODE:-repo}"
+PIPELINE_TEMPLATE="${PIPELINE_TEMPLATE:-autonomous}"
+ISSUE_LIMIT="${ISSUE_LIMIT:-100}"
+SLACK_WEBHOOK="${SLACK_WEBHOOK:-}"
+BACKOFF_SECS="${BACKOFF_SECS:-0}"
+POLL_CYCLE_COUNT="${POLL_CYCLE_COUNT:-0}"
+
+daemon_health_check() {
+    local findings=0
+    local now_e
+    now_e=$(now_epoch)
+
+    if [[ -f "$STATE_FILE" ]]; then
+        # ── Intelligent Health Monitoring ──
+        # Instead of killing after a countdown, sense what the agent is doing.
+        # Agents think for long stretches — that's normal and expected.
+        # Strategy: sense → understand → be patient → nudge → only kill as last resort.
+
+        local hard_limit="${PROGRESS_HARD_LIMIT_S:-0}"
+        local use_progress="${PROGRESS_MONITORING:-true}"
+        local nudge_enabled="${NUDGE_ENABLED:-true}"
+        local nudge_after="${NUDGE_AFTER_CHECKS:-40}"
+
+        while IFS= read -r job; do
+            local pid started_at issue_num worktree
+            pid=$(echo "$job" | jq -r '.pid')
+            started_at=$(echo "$job" | jq -r '.started_at // empty')
+            issue_num=$(echo "$job" | jq -r '.issue')
+            worktree=$(echo "$job" | jq -r '.worktree // ""')
+
+            # Skip dead processes
+            if ! kill -0 "$pid" 2>/dev/null; then
+                continue
+            fi
+
+            local elapsed=0
+            if [[ -n "$started_at" ]]; then
+                local start_e
+                start_e=$(TZ=UTC date -j -f "%Y-%m-%dT%H:%M:%SZ" "$started_at" +%s 2>/dev/null || date -d "$started_at" +%s 2>/dev/null || echo "0")
+                elapsed=$(( now_e - start_e ))
+            fi
+
+            # Hard wall-clock limit — disabled by default (0 = off)
+            if [[ "$hard_limit" -gt 0 && "$elapsed" -gt "$hard_limit" ]]; then
+                daemon_log WARN "Hard limit exceeded: issue #${issue_num} (${elapsed}s > ${hard_limit}s, PID $pid) — killing"
+                emit_event "daemon.hard_limit" "issue=$issue_num" "elapsed_s=$elapsed" "limit_s=$hard_limit" "pid=$pid"
+                kill "$pid" 2>/dev/null || true
+                daemon_clear_progress "$issue_num"
+                findings=$((findings + 1))
+                continue
+            fi
+
+            # ── Intelligent Progress Sensing ──
+            if [[ "$use_progress" == "true" && -n "$worktree" ]]; then
+                local snapshot verdict
+                snapshot=$(daemon_collect_snapshot "$issue_num" "$worktree" "$pid" 2>/dev/null || echo '{}')
+
+                if [[ "$snapshot" != "{}" ]]; then
+                    verdict=$(daemon_assess_progress "$issue_num" "$snapshot" 2>/dev/null || echo "healthy")
+
+                    local no_progress_count=0
+                    no_progress_count=$(jq -r '.no_progress_count // 0' "$PROGRESS_DIR/issue-${issue_num}.json" 2>/dev/null || echo 0)
+                    local cur_stage
+                    cur_stage=$(echo "$snapshot" | jq -r '.stage // "unknown"')
+
+                    case "$verdict" in
+                        healthy)
+                            # All good — agent is making progress
+                            ;;
+                        slowing)
+                            daemon_log INFO "Issue #${issue_num} slowing (no visible changes for ${no_progress_count} checks, ${elapsed}s elapsed, stage=${cur_stage})"
+                            ;;
+                        stalled)
+                            # Check if agent subprocess is alive and consuming CPU
+                            local agent_alive=false
+                            local child_cpu=0
+                            child_cpu=$(pgrep -P "$pid" 2>/dev/null | xargs -I{} ps -o pcpu= -p {} 2>/dev/null | awk '{sum+=$1} END{printf "%d", sum+0}' || echo "0")
+                            if [[ "${child_cpu:-0}" -gt 0 ]]; then
+                                agent_alive=true
+                            fi
+
+                            if [[ "$agent_alive" == "true" ]]; then
+                                daemon_log INFO "Issue #${issue_num} no visible progress (${no_progress_count} checks) but agent is alive (CPU: ${child_cpu}%, stage=${cur_stage}, ${elapsed}s) — being patient"
+                            else
+                                daemon_log WARN "Issue #${issue_num} stalled: no progress for ${no_progress_count} checks, no CPU activity (${elapsed}s elapsed, PID $pid)"
+                                emit_event "daemon.stalled" "issue=$issue_num" "no_progress=$no_progress_count" "elapsed_s=$elapsed" "pid=$pid"
+                            fi
+                            ;;
+                        stuck)
+                            local repeated_errors
+                            repeated_errors=$(jq -r '.repeated_error_count // 0' "$PROGRESS_DIR/issue-${issue_num}.json" 2>/dev/null || echo 0)
+
+                            # Even "stuck" — check if the process tree is alive first
+                            local agent_alive=false
+                            local child_cpu=0
+                            child_cpu=$(pgrep -P "$pid" 2>/dev/null | xargs -I{} ps -o pcpu= -p {} 2>/dev/null | awk '{sum+=$1} END{printf "%d", sum+0}' || echo "0")
+                            if [[ "${child_cpu:-0}" -gt 0 ]]; then
+                                agent_alive=true
+                            fi
+
+                            if [[ "$agent_alive" == "true" && "$repeated_errors" -lt 3 ]]; then
+                                # Agent is alive — nudge instead of kill
+                                if [[ "$nudge_enabled" == "true" && "$no_progress_count" -ge "$nudge_after" ]]; then
+                                    local nudge_file="${worktree}/.claude/nudge.md"
+                                    if [[ ! -f "$nudge_file" ]]; then
+                                        cat > "$nudge_file" <<NUDGE_EOF
+# Nudge from Daemon Health Monitor
+
+The daemon has noticed no visible progress for $(( no_progress_count * 30 / 60 )) minutes.
+Current stage: ${cur_stage}
+
+If you're stuck, consider:
+- Breaking the task into smaller steps
+- Committing partial progress
+- Running tests to validate current state
+
+This is just a gentle check-in — take your time if you're working through a complex problem.
+NUDGE_EOF
+                                        daemon_log INFO "Issue #${issue_num} nudged (${no_progress_count} checks, stage=${cur_stage}, CPU=${child_cpu}%) — file written to worktree"
+                                        emit_event "daemon.nudge" "issue=$issue_num" "no_progress=$no_progress_count" "stage=$cur_stage" "elapsed_s=$elapsed"
+                                    fi
+                                else
+                                    daemon_log INFO "Issue #${issue_num} no visible progress (${no_progress_count} checks) but agent is alive (CPU: ${child_cpu}%, stage=${cur_stage}) — waiting"
+                                fi
+                            elif [[ "$repeated_errors" -ge 5 ]]; then
+                                # Truly stuck in an error loop — kill as last resort
+                                daemon_log WARN "Issue #${issue_num} in error loop: ${repeated_errors} repeated errors (stage=${cur_stage}, ${elapsed}s, PID $pid) — killing"
+                                emit_event "daemon.stuck_kill" "issue=$issue_num" "no_progress=$no_progress_count" "repeated_errors=$repeated_errors" "stage=$cur_stage" "elapsed_s=$elapsed" "pid=$pid" "reason=error_loop"
+                                kill "$pid" 2>/dev/null || true
+                                daemon_clear_progress "$issue_num"
+                                findings=$((findings + 1))
+                            elif [[ "$agent_alive" != "true" && "$no_progress_count" -ge "$((PROGRESS_CHECKS_BEFORE_KILL * 2))" ]]; then
+                                # Process tree is dead AND no progress for very long time
+                                daemon_log WARN "Issue #${issue_num} appears dead: no CPU, no progress for ${no_progress_count} checks (${elapsed}s, PID $pid) — killing"
+                                emit_event "daemon.stuck_kill" "issue=$issue_num" "no_progress=$no_progress_count" "repeated_errors=$repeated_errors" "stage=$cur_stage" "elapsed_s=$elapsed" "pid=$pid" "reason=dead_process"
+                                kill "$pid" 2>/dev/null || true
+                                daemon_clear_progress "$issue_num"
+                                findings=$((findings + 1))
+                            else
+                                daemon_log WARN "Issue #${issue_num} struggling (${no_progress_count} checks, ${repeated_errors} errors, CPU=${child_cpu}%, stage=${cur_stage}) — monitoring"
+                            fi
+                            ;;
+                    esac
+                fi
+            else
+                # Fallback: legacy time-based detection when progress monitoring is off
+                local stale_timeout
+                stale_timeout=$(get_adaptive_stale_timeout "$PIPELINE_TEMPLATE")
+                if [[ "$elapsed" -gt "$stale_timeout" ]]; then
+                    # Check if process is still alive
+                    if kill -0 "$pid" 2>/dev/null; then
+                        # Kill at 2x stale timeout — the process is truly hung
+                        local kill_threshold=$(( stale_timeout * 2 ))
+                        if [[ "$elapsed" -gt "$kill_threshold" ]]; then
+                            daemon_log WARN "Killing stale job (legacy): issue #${issue_num} (${elapsed}s > ${kill_threshold}s kill threshold, PID $pid)"
+                            emit_event "daemon.stale_kill" "issue=$issue_num" "elapsed_s=$elapsed" "pid=$pid"
+                            kill "$pid" 2>/dev/null || true
+                            sleep 2
+                            kill -9 "$pid" 2>/dev/null || true
+                        else
+                            daemon_log WARN "Stale job (legacy): issue #${issue_num} (${elapsed}s > ${stale_timeout}s, PID $pid) — will kill at ${kill_threshold}s"
+                            emit_event "daemon.stale_warning" "issue=$issue_num" "elapsed_s=$elapsed" "pid=$pid"
+                        fi
+                    else
+                        daemon_log WARN "Stale job with dead process: issue #${issue_num} (PID $pid no longer exists)"
+                        emit_event "daemon.stale_dead" "issue=$issue_num" "pid=$pid"
+                    fi
+                    findings=$((findings + 1))
+                fi
+            fi
+        done < <(jq -c '.active_jobs[]' "$STATE_FILE" 2>/dev/null || true)
+    fi
+
+    # Disk space warning (check both repo dir and ~/.shipwright)
+    local free_kb
+    free_kb=$(df -k "." 2>/dev/null | tail -1 | awk '{print $4}')
+    if [[ -n "$free_kb" ]] && [[ "$free_kb" -lt 1048576 ]] 2>/dev/null; then
+        daemon_log WARN "Low disk space: $(( free_kb / 1024 ))MB free"
+        findings=$((findings + 1))
+    fi
+
+    # Critical disk space on ~/.shipwright — pause spawning
+    local sw_free_kb
+    sw_free_kb=$(df -k "$HOME/.shipwright" 2>/dev/null | tail -1 | awk '{print $4}')
+    if [[ -n "$sw_free_kb" ]] && [[ "$sw_free_kb" -lt 512000 ]] 2>/dev/null; then
+        daemon_log WARN "Critical disk space on ~/.shipwright: $(( sw_free_kb / 1024 ))MB — pausing spawns"
+        emit_event "daemon.disk_low" "free_mb=$(( sw_free_kb / 1024 ))"
+        mkdir -p "$HOME/.shipwright"
+        echo '{"paused":true,"reason":"disk_low"}' > "$HOME/.shipwright/daemon-pause.flag"
+        findings=$((findings + 1))
+    fi
+
+    # Events file size warning
+    if [[ -f "$EVENTS_FILE" ]]; then
+        local events_size
+        events_size=$(wc -c < "$EVENTS_FILE" 2>/dev/null || echo 0)
+        if [[ "$events_size" -gt 104857600 ]]; then  # 100MB
+            daemon_log WARN "Events file large ($(( events_size / 1048576 ))MB) — consider rotating"
+            findings=$((findings + 1))
+        fi
+    fi
+
+    if [[ "$findings" -gt 0 ]]; then
+        emit_event "daemon.health" "findings=$findings"
+    fi
+}
+
+# ─── Degradation Alerting ─────────────────────────────────────────────────────
+
+
+daemon_check_degradation() {
+    if [[ ! -f "$EVENTS_FILE" ]]; then return; fi
+
+    local window="${DEGRADATION_WINDOW:-5}"
+    local cfr_threshold="${DEGRADATION_CFR_THRESHOLD:-30}"
+    local success_threshold="${DEGRADATION_SUCCESS_THRESHOLD:-50}"
+
+    # Get last N pipeline completions
+    local recent
+    recent=$(tail -200 "$EVENTS_FILE" | jq -s "[.[] | select(.type == \"pipeline.completed\")] | .[-${window}:]" 2>/dev/null)
+    local count
+    count=$(echo "$recent" | jq 'length' 2>/dev/null || echo 0)
+
+    if [[ "$count" -lt "$window" ]]; then return; fi
+
+    local failures successes
+    failures=$(echo "$recent" | jq '[.[] | select(.result == "failure")] | length')
+    successes=$(echo "$recent" | jq '[.[] | select(.result == "success")] | length')
+    local cfr_pct=0 success_pct=0
+    if [[ "${count:-0}" -gt 0 ]]; then
+        cfr_pct=$(( failures * 100 / count ))
+        success_pct=$(( successes * 100 / count ))
+    fi
+
+    local alerts=""
+    if [[ "$cfr_pct" -gt "$cfr_threshold" ]]; then
+        alerts="CFR ${cfr_pct}% exceeds threshold ${cfr_threshold}%"
+        daemon_log WARN "DEGRADATION: $alerts"
+    fi
+    if [[ "$success_pct" -lt "$success_threshold" ]]; then
+        local msg="Success rate ${success_pct}% below threshold ${success_threshold}%"
+        [[ -n "$alerts" ]] && alerts="$alerts; $msg" || alerts="$msg"
+        daemon_log WARN "DEGRADATION: $msg"
+    fi
+
+    if [[ -n "$alerts" ]]; then
+        emit_event "daemon.alert" "alerts=$alerts" "cfr_pct=$cfr_pct" "success_pct=$success_pct"
+
+        # Slack notification
+        if [[ -n "${SLACK_WEBHOOK:-}" ]]; then
+            notify "Pipeline Degradation Alert" "$alerts" "warn"
+        fi
+    fi
+}
+
+# ─── Auto-Scaling ─────────────────────────────────────────────────────────
+# Dynamically adjusts MAX_PARALLEL based on CPU, memory, budget, and queue depth
+
+
+daemon_reload_config() {
+    local reload_flag="$HOME/.shipwright/fleet-reload.flag"
+    if [[ ! -f "$reload_flag" ]]; then
+        return
+    fi
+
+    local fleet_config=".claude/.fleet-daemon-config.json"
+    if [[ -f "$fleet_config" ]]; then
+        local new_max
+        new_max=$(jq -r '.max_parallel // empty' "$fleet_config" 2>/dev/null || true)
+        if [[ -n "$new_max" && "$new_max" != "null" ]]; then
+            local prev="$MAX_PARALLEL"
+            FLEET_MAX_PARALLEL="$new_max"
+            MAX_PARALLEL="$new_max"
+            daemon_log INFO "Fleet reload: max_parallel ${prev} → ${MAX_PARALLEL} (fleet ceiling: ${FLEET_MAX_PARALLEL})"
+            emit_event "daemon.fleet_reload" "from=$prev" "to=$MAX_PARALLEL"
+        fi
+    fi
+
+    rm -f "$reload_flag"
+}
+
+# ─── Self-Optimizing Metrics Loop ──────────────────────────────────────────
+