shipwright-cli 3.2.0 → 3.3.0

package/scripts/lib/pipeline-execution.sh

@@ -0,0 +1,897 @@
+#!/usr/bin/env bash
+# Module: pipeline-execution
+# Execution orchestration: stage retry logic, self-healing build-test loop, main pipeline run
+set -euo pipefail
+
+# Module guard
+[[ -n "${_MODULE_PIPELINE_EXECUTION_LOADED:-}" ]] && return 0; _MODULE_PIPELINE_EXECUTION_LOADED=1
+
+# ─── Defaults (needed if sourced independently) ──────────────────────────────
+SCRIPT_DIR="${SCRIPT_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)}"
+REPO_DIR="${REPO_DIR:-$(cd "$SCRIPT_DIR/.." && pwd)}"
+PROJECT_ROOT="${PROJECT_ROOT:-$(git rev-parse --show-toplevel 2>/dev/null || pwd)}"
+STATE_DIR="${STATE_DIR:-$PROJECT_ROOT/.claude}"
+STATE_FILE="${STATE_FILE:-$STATE_DIR/pipeline-state.md}"
+ARTIFACTS_DIR="${ARTIFACTS_DIR:-$STATE_DIR/pipeline-artifacts}"
+
+# Variables referenced by execution functions (set by sw-pipeline.sh, defaults here for safety)
+BUILD_TEST_RETRIES="${BUILD_TEST_RETRIES:-2}"
+SELF_HEAL_COUNT="${SELF_HEAL_COUNT:-0}"
+STASHED_CHANGES="${STASHED_CHANGES:-false}"
+NOTIFICATION_ENABLED="${NOTIFICATION_ENABLED:-false}"
+HEARTBEAT_PID="${HEARTBEAT_PID:-}"
+
+# Ensure helpers are loaded
+[[ -f "$SCRIPT_DIR/lib/helpers.sh" ]] && source "$SCRIPT_DIR/lib/helpers.sh" 2>/dev/null || true
+[[ "$(type -t info 2>/dev/null)" == "function" ]] || info() { echo "$*"; }
+[[ "$(type -t warn 2>/dev/null)" == "function" ]] || warn() { echo "$*"; }
+[[ "$(type -t error 2>/dev/null)" == "function" ]] || error() { echo "$*" >&2; }
+[[ "$(type -t emit_event 2>/dev/null)" == "function" ]] || emit_event() { true; }
+
+# Ensure pipeline intelligence skip module is loaded (provides pipeline_should_skip_stage)
+# SCRIPT_DIR may point to scripts/ or scripts/lib/ depending on how this module was sourced
+if [[ -f "$SCRIPT_DIR/pipeline-intelligence-skip.sh" ]]; then
+    source "$SCRIPT_DIR/pipeline-intelligence-skip.sh" 2>/dev/null || true
+elif [[ -f "$SCRIPT_DIR/lib/pipeline-intelligence-skip.sh" ]]; then
+    source "$SCRIPT_DIR/lib/pipeline-intelligence-skip.sh" 2>/dev/null || true
+fi
+
+# ─── Stage Execution with Retry Logic ──────────────────────────────
+run_stage_with_retry() {
+    local stage_id="$1"
+    local max_retries
+    max_retries=$(jq -r --arg id "$stage_id" '(.stages[] | select(.id == $id) | .config.retries) // 0' "$PIPELINE_CONFIG" 2>/dev/null) || true
+    [[ -z "$max_retries" || "$max_retries" == "null" ]] && max_retries=0
+
+    local attempt=0
+    local prev_error_class=""
+    while true; do
+        if "stage_${stage_id}"; then
+            return 0
+        fi
+
+        # Capture error_class and error snippet for stage.failed / pipeline.completed events
+        local error_class
+        error_class=$(classify_error "$stage_id")
+        LAST_STAGE_ERROR_CLASS="$error_class"
+        LAST_STAGE_ERROR=""
+        local _log_file="${ARTIFACTS_DIR}/${stage_id}-results.log"
+        [[ ! -f "$_log_file" ]] && _log_file="${ARTIFACTS_DIR}/test-results.log"
+        if [[ -f "$_log_file" ]]; then
+            LAST_STAGE_ERROR=$(tail -20 "$_log_file" 2>/dev/null | grep -iE 'error|fail|exception|fatal' 2>/dev/null | head -1 | cut -c1-200 || true)
+        fi
+
+        attempt=$((attempt + 1))
+
+        # Critical fix: if plan stage already has a valid artifact, skip retry
+        if [[ "$stage_id" == "plan" ]]; then
+            local plan_artifact="${ARTIFACTS_DIR}/plan.md"
+            if [[ -s "$plan_artifact" ]]; then
+                local existing_lines
+                existing_lines=$(wc -l < "$plan_artifact" 2>/dev/null | xargs)
+                existing_lines="${existing_lines:-0}"
+                if [[ "$existing_lines" -gt 10 ]]; then
+                    info "Plan already exists (${existing_lines} lines) — skipping retry, advancing"
+                    emit_event "retry.skipped_existing_artifact" \
+                        "issue=${ISSUE_NUMBER:-0}" \
+                        "stage=$stage_id" \
+                        "artifact_lines=$existing_lines"
+                    return 0
+                fi
+            fi
+        fi
+
+        if [[ "$attempt" -gt "$max_retries" ]]; then
+            return 1
+        fi
+
+        # Classify done above; decide whether retry makes sense
+
+        emit_event "retry.classified" \
+            "issue=${ISSUE_NUMBER:-0}" \
+            "stage=$stage_id" \
+            "attempt=$attempt" \
+            "error_class=$error_class"
+
+        case "$error_class" in
+            infrastructure)
+                info "Error classified as infrastructure (timeout/network/OOM) — retry makes sense"
+                ;;
+            configuration)
+                error "Error classified as configuration (missing env/path) — skipping retry, escalating"
+                emit_event "retry.escalated" \
+                    "issue=${ISSUE_NUMBER:-0}" \
+                    "stage=$stage_id" \
+                    "reason=configuration_error"
+                return 1
+                ;;
+            logic)
+                if [[ "$error_class" == "$prev_error_class" ]]; then
+                    error "Error classified as logic (assertion/type error) with same class — retry won't help without code change"
+                    emit_event "retry.skipped" \
+                        "issue=${ISSUE_NUMBER:-0}" \
+                        "stage=$stage_id" \
+                        "reason=repeated_logic_error"
+                    return 1
+                fi
+                warn "Error classified as logic — retrying once in case build fixes it"
+                ;;
+            *)
+                info "Error classification: unknown — retrying"
+                ;;
+        esac
+        prev_error_class="$error_class"
+
+        if type db_save_reasoning_trace >/dev/null 2>&1; then
+            local job_id="${SHIPWRIGHT_PIPELINE_ID:-$$}"
+            local error_msg="${LAST_STAGE_ERROR:-$error_class}"
+            db_save_reasoning_trace "$job_id" "retry_reasoning" \
+                "stage=$stage_id error=$error_msg" \
+                "Stage failed, analyzing error pattern before retry" \
+                "retry_strategy=self_heal" 0.6 2>/dev/null || true
+        fi
+
+        warn "Stage $stage_id failed (attempt $attempt/$((max_retries + 1)), class: $error_class) — retrying..."
+        # Exponential backoff with jitter to avoid thundering herd
+        local backoff=$((2 ** attempt))
+        [[ "$backoff" -gt 16 ]] && backoff=16
+        local jitter=$(( RANDOM % (backoff + 1) ))
+        local total_sleep=$((backoff + jitter))
+        info "Backing off ${total_sleep}s before retry..."
+        sleep "$total_sleep"
+
+        # Write debugging context for the retry attempt to consume
+        local _retry_ctx_file="${ARTIFACTS_DIR}/.retry-context-${stage_id}.md"
+        {
+            echo "## Previous Attempt Failed"
+            echo ""
+            echo "**Error classification:** ${error_class}"
+            echo "**Attempt:** ${attempt} of $((max_retries + 1))"
+            echo ""
+            echo "### Error Output (last 30 lines)"
+            echo '```'
+            tail -30 "$_log_file" 2>/dev/null || echo "(no log available)"
+            echo '```'
+            echo ""
+            # Check for existing artifacts that should be preserved
+            local _existing_artifacts=""
+            for _af in plan.md design.md test-results.log; do
+                if [[ -s "${ARTIFACTS_DIR}/${_af}" ]]; then
+                    local _af_lines
+                    _af_lines=$(wc -l < "${ARTIFACTS_DIR}/${_af}" 2>/dev/null | xargs)
+                    _existing_artifacts="${_existing_artifacts}  - ${_af} (${_af_lines} lines)\n"
+                fi
+            done
+            if [[ -n "$_existing_artifacts" ]]; then
+                echo "### Existing Artifacts (PRESERVE these)"
+                echo -e "$_existing_artifacts"
+                echo "These artifacts exist from previous successful stages. Use them as-is unless they are the source of the problem."
+                echo ""
+            fi
+            # Adaptive: check if additional skills could help this retry
+            if type skill_memory_get_recommendations >/dev/null 2>&1; then
+                local _retry_skills
+                _retry_skills=$(skill_memory_get_recommendations "${INTELLIGENCE_ISSUE_TYPE:-backend}" "$stage_id" 2>/dev/null || true)
+                if [[ -n "$_retry_skills" ]]; then
+                    echo "### Skills Recommended by Learning System"
+                    echo "Based on historical success rates, these skills may improve the retry:"
+                    echo "- $(printf '%s' "$_retry_skills" | sed 's/,/\n- /g')"
+                    echo ""
+                fi
+            fi
+
+            echo "### Investigation Required"
+            echo "Before attempting a fix:"
+            echo "1. Read the error output above carefully"
+            echo "2. Identify the ROOT CAUSE — not just the symptom"
+            echo "3. If previous artifacts exist and are correct, build on them"
+            echo "4. If previous artifacts are flawed, explain what's wrong before fixing"
+        } > "$_retry_ctx_file" 2>/dev/null || true
+
+        emit_event "retry.context_written" \
+            "issue=${ISSUE_NUMBER:-0}" \
+            "stage=$stage_id" \
+            "attempt=$attempt" \
+            "context_file=$_retry_ctx_file"
+    done
+}
+
+# ─── Self-Healing Build→Test Feedback Loop ─────────────────────────
+self_healing_build_test() {
+    local cycle=0
+    local max_cycles="$BUILD_TEST_RETRIES"
+    local last_test_error=""
+
+    # Convergence tracking
+    local prev_error_sig="" consecutive_same_error=0
+    local prev_fail_count=0 zero_convergence_streak=0
+
+    # Vitals-driven adaptive limit (preferred over static BUILD_TEST_RETRIES)
+    if type pipeline_adaptive_limit >/dev/null 2>&1; then
+        local _vitals_json=""
+        if type pipeline_compute_vitals >/dev/null 2>&1; then
+            _vitals_json=$(pipeline_compute_vitals "$STATE_FILE" "$ARTIFACTS_DIR" "${ISSUE_NUMBER:-}" 2>/dev/null) || true
+        fi
+        local vitals_limit
+        vitals_limit=$(pipeline_adaptive_limit "build_test" "$_vitals_json" 2>/dev/null) || true
+        if [[ -n "$vitals_limit" && "$vitals_limit" =~ ^[0-9]+$ && "$vitals_limit" -gt 0 ]]; then
+            info "Vitals-driven build-test limit: ${max_cycles} → ${vitals_limit}"
+            max_cycles="$vitals_limit"
+            emit_event "vitals.adaptive_limit" \
+                "issue=${ISSUE_NUMBER:-0}" \
+                "context=build_test" \
+                "original=$BUILD_TEST_RETRIES" \
+                "vitals_limit=$vitals_limit"
+        fi
+    # Fallback: intelligence-based adaptive limits
+    elif type composer_estimate_iterations >/dev/null 2>&1; then
+        local estimated
+        estimated=$(composer_estimate_iterations \
+            "${INTELLIGENCE_ANALYSIS:-{}}" \
+            "${HOME}/.shipwright/optimization/iteration-model.json" 2>/dev/null || echo "")
+        if [[ -n "$estimated" && "$estimated" =~ ^[0-9]+$ && "$estimated" -gt 0 ]]; then
+            max_cycles="$estimated"
+            emit_event "intelligence.adaptive_iterations" \
+                "issue=${ISSUE_NUMBER:-0}" \
+                "estimated=$estimated" \
+                "original=$BUILD_TEST_RETRIES"
+        fi
+    fi
+
+    # Fallback: adaptive cycle limits from optimization data
+    if [[ "$max_cycles" == "$BUILD_TEST_RETRIES" ]]; then
+        local _iter_model="${HOME}/.shipwright/optimization/iteration-model.json"
+        if [[ -f "$_iter_model" ]]; then
+            local adaptive_bt_limit
+            adaptive_bt_limit=$(pipeline_adaptive_cycles "$max_cycles" "build_test" "0" "-1" 2>/dev/null) || true
+            if [[ -n "$adaptive_bt_limit" && "$adaptive_bt_limit" =~ ^[0-9]+$ && "$adaptive_bt_limit" -gt 0 && "$adaptive_bt_limit" != "$max_cycles" ]]; then
+                info "Adaptive build-test cycles: ${max_cycles} → ${adaptive_bt_limit}"
+                max_cycles="$adaptive_bt_limit"
+            fi
+        fi
+    fi
+
+    while [[ "$cycle" -le "$max_cycles" ]]; do
+        cycle=$((cycle + 1))
+
+        if [[ "$cycle" -gt 1 ]]; then
+            SELF_HEAL_COUNT=$((SELF_HEAL_COUNT + 1))
+            echo ""
+            echo -e "${YELLOW}${BOLD}━━━ Self-Healing Cycle ${cycle}/$((max_cycles + 1)) ━━━${RESET}"
+            info "Feeding test failure back to build loop..."
+
+            if [[ -n "$ISSUE_NUMBER" ]]; then
+                gh_comment_issue "$ISSUE_NUMBER" "🔄 **Self-healing cycle ${cycle}** — rebuilding with error context" 2>/dev/null || true
+            fi
+
+            # Reset build/test stage statuses for retry
+            set_stage_status "build" "retrying"
+            set_stage_status "test" "pending"
+        fi
+
+        # ── Run Build Stage ──
+        echo ""
+        echo -e "${CYAN}${BOLD}▸ Stage: build${RESET} ${DIM}[cycle ${cycle}]${RESET}"
+        CURRENT_STAGE_ID="build"
+
+        # Inject error context on retry cycles
+        if [[ "$cycle" -gt 1 && -n "$last_test_error" ]]; then
+            # Query memory for known fixes
+            local _memory_fix=""
+            if type memory_closed_loop_inject >/dev/null 2>&1; then
+                local _error_sig_short
+                _error_sig_short=$(echo "$last_test_error" | head -3 || echo "")
+                _memory_fix=$(memory_closed_loop_inject "$_error_sig_short" 2>/dev/null) || true
+            fi
+
+            local memory_prefix=""
+            if [[ -n "$_memory_fix" ]]; then
+                info "Memory suggests fix: $(echo "$_memory_fix" | head -1)"
+                memory_prefix="KNOWN FIX (from past success): ${_memory_fix}
+
+"
+            fi
+
+            # Temporarily augment the goal with error context
+            local original_goal="$GOAL"
+            GOAL="$GOAL
+
+${memory_prefix}IMPORTANT — Previous build attempt failed tests. Fix these errors:
+$last_test_error
+
+Focus on fixing the failing tests while keeping all passing tests working."
+
+            update_status "running" "build"
+            record_stage_start "build"
+            type audit_emit >/dev/null 2>&1 && audit_emit "stage.start" "stage=build" || true
+
+            local build_start_epoch
+            build_start_epoch=$(date +%s)
+            if run_stage_with_retry "build"; then
+                mark_stage_complete "build"
+                local timing
+                timing=$(get_stage_timing "build")
+                local build_dur_s=$(( $(date +%s) - build_start_epoch ))
+                type audit_emit >/dev/null 2>&1 && audit_emit "stage.complete" "stage=build" "verdict=pass" "duration_s=${build_dur_s}" || true
+                success "Stage ${BOLD}build${RESET} complete ${DIM}(${timing})${RESET}"
+                if type pipeline_emit_progress_snapshot >/dev/null 2>&1 && [[ -n "${ISSUE_NUMBER:-}" ]]; then
+                    local _diff_count
+                    _diff_count=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1) || true
+                    local _snap_files _snap_error
+                    _snap_files=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1 || true)
+                    _snap_files="${_snap_files:-0}"
+                    _snap_error=$(tail -1 "$ARTIFACTS_DIR/error-log.jsonl" 2>/dev/null | jq -r '.error // ""' 2>/dev/null || true)
+                    _snap_error="${_snap_error:-}"
+                    pipeline_emit_progress_snapshot "${ISSUE_NUMBER}" "${CURRENT_STAGE_ID:-build}" "${cycle:-0}" "${_diff_count:-0}" "${_snap_files}" "${_snap_error}" 2>/dev/null || true
+                fi
+            else
+                mark_stage_failed "build"
+                local build_dur_s=$(( $(date +%s) - build_start_epoch ))
+                type audit_emit >/dev/null 2>&1 && audit_emit "stage.complete" "stage=build" "verdict=fail" "duration_s=${build_dur_s}" || true
+                GOAL="$original_goal"
+                return 1
+            fi
+            GOAL="$original_goal"
+        else
+            update_status "running" "build"
+            record_stage_start "build"
+            type audit_emit >/dev/null 2>&1 && audit_emit "stage.start" "stage=build" || true
+
+            local build_start_epoch
+            build_start_epoch=$(date +%s)
+            if run_stage_with_retry "build"; then
+                mark_stage_complete "build"
+                local timing
+                timing=$(get_stage_timing "build")
+                local build_dur_s=$(( $(date +%s) - build_start_epoch ))
+                type audit_emit >/dev/null 2>&1 && audit_emit "stage.complete" "stage=build" "verdict=pass" "duration_s=${build_dur_s}" || true
+                success "Stage ${BOLD}build${RESET} complete ${DIM}(${timing})${RESET}"
+                if type pipeline_emit_progress_snapshot >/dev/null 2>&1 && [[ -n "${ISSUE_NUMBER:-}" ]]; then
+                    local _diff_count
+                    _diff_count=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1) || true
+                    local _snap_files _snap_error
+                    _snap_files=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1 || true)
+                    _snap_files="${_snap_files:-0}"
+                    _snap_error=$(tail -1 "$ARTIFACTS_DIR/error-log.jsonl" 2>/dev/null | jq -r '.error // ""' 2>/dev/null || true)
+                    _snap_error="${_snap_error:-}"
+                    pipeline_emit_progress_snapshot "${ISSUE_NUMBER}" "${CURRENT_STAGE_ID:-build}" "${cycle:-0}" "${_diff_count:-0}" "${_snap_files}" "${_snap_error}" 2>/dev/null || true
+                fi
+            else
+                mark_stage_failed "build"
+                local build_dur_s=$(( $(date +%s) - build_start_epoch ))
+                type audit_emit >/dev/null 2>&1 && audit_emit "stage.complete" "stage=build" "verdict=fail" "duration_s=${build_dur_s}" || true
+                return 1
+            fi
+        fi
+
+        # ── Run Test Stage ──
+        echo ""
+        echo -e "${CYAN}${BOLD}▸ Stage: test${RESET} ${DIM}[cycle ${cycle}]${RESET}"
+        CURRENT_STAGE_ID="test"
+        update_status "running" "test"
+        record_stage_start "test"
+
+        if run_stage_with_retry "test"; then
+            mark_stage_complete "test"
+            local timing
+            timing=$(get_stage_timing "test")
+            success "Stage ${BOLD}test${RESET} complete ${DIM}(${timing})${RESET}"
+            emit_event "convergence.tests_passed" \
+                "issue=${ISSUE_NUMBER:-0}" \
+                "cycle=$cycle"
+            if type pipeline_emit_progress_snapshot >/dev/null 2>&1 && [[ -n "${ISSUE_NUMBER:-}" ]]; then
+                local _diff_count
+                _diff_count=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1) || true
+                local _snap_files _snap_error
+                _snap_files=$(git diff --stat HEAD~1 2>/dev/null | tail -1 | grep -oE '[0-9]+' | head -1 || true)
+                _snap_files="${_snap_files:-0}"
+                _snap_error=$(tail -1 "$ARTIFACTS_DIR/error-log.jsonl" 2>/dev/null | jq -r '.error // ""' 2>/dev/null || true)
+                _snap_error="${_snap_error:-}"
+                pipeline_emit_progress_snapshot "${ISSUE_NUMBER}" "${CURRENT_STAGE_ID:-test}" "${cycle:-0}" "${_diff_count:-0}" "${_snap_files}" "${_snap_error}" 2>/dev/null || true
+            fi
+            # Record fix outcome when tests pass after a retry with memory injection (pipeline path)
+            if [[ "$cycle" -gt 1 && -n "${last_test_error:-}" ]] && [[ -x "$SCRIPT_DIR/sw-memory.sh" ]]; then
+                local _sig
+                _sig=$(echo "$last_test_error" | head -3 | tr '\n' ' ' | sed 's/^ *//;s/ *$//')
+                [[ -n "$_sig" ]] && bash "$SCRIPT_DIR/sw-memory.sh" fix-outcome "$_sig" "true" "true" 2>/dev/null || true
+            fi
+            return 0  # Tests passed!
+        fi
+
+        # Tests failed — capture error for next cycle
+        local test_log="$ARTIFACTS_DIR/test-results.log"
+        last_test_error=$(tail -30 "$test_log" 2>/dev/null || echo "Test command failed with no output")
+        mark_stage_failed "test"
+
+        # ── Convergence Detection ──
+        # Hash the error output to detect repeated failures
+        local error_sig
+        error_sig=$(echo "$last_test_error" | shasum -a 256 2>/dev/null | cut -c1-16 || echo "unknown")
+
+        # Count failing tests (extract from common patterns)
+        local current_fail_count=0
+        current_fail_count=$(grep -ciE 'fail|error|FAIL' "$test_log" 2>/dev/null || true)
+        current_fail_count="${current_fail_count:-0}"
+
+        if [[ "$error_sig" == "$prev_error_sig" ]]; then
+            consecutive_same_error=$((consecutive_same_error + 1))
+        else
+            consecutive_same_error=1
+        fi
+        prev_error_sig="$error_sig"
+
+        # Check: same error 3 times consecutively → stuck
+        if [[ "$consecutive_same_error" -ge 3 ]]; then
+            error "Convergence: stuck on same error for 3 consecutive cycles — exiting early"
+            emit_event "convergence.stuck" \
+                "issue=${ISSUE_NUMBER:-0}" \
+                "cycle=$cycle" \
+                "error_sig=$error_sig" \
+                "consecutive=$consecutive_same_error"
+            notify "Build Convergence" "Stuck on unfixable error after ${cycle} cycles" "error"
+            return 1
+        fi
+
+        # Track convergence rate: did we reduce failures?
+        if [[ "$cycle" -gt 1 && "$prev_fail_count" -gt 0 ]]; then
+            if [[ "$current_fail_count" -ge "$prev_fail_count" ]]; then
+                zero_convergence_streak=$((zero_convergence_streak + 1))
+            else
+                zero_convergence_streak=0
+            fi
+
+            # Check: zero convergence for 2 consecutive iterations → plateau
+            if [[ "$zero_convergence_streak" -ge 2 ]]; then
+                error "Convergence: no progress for 2 consecutive cycles (${current_fail_count} failures remain) — exiting early"
+                emit_event "convergence.plateau" \
+                    "issue=${ISSUE_NUMBER:-0}" \
+                    "cycle=$cycle" \
+                    "fail_count=$current_fail_count" \
+                    "streak=$zero_convergence_streak"
+                notify "Build Convergence" "No progress after ${cycle} cycles — plateau reached" "error"
+                return 1
+            fi
+        fi
+        prev_fail_count="$current_fail_count"
+
+        info "Convergence: error_sig=${error_sig:0:8} repeat=${consecutive_same_error} failures=${current_fail_count} no_progress=${zero_convergence_streak}"
+
+        if [[ "$cycle" -le "$max_cycles" ]]; then
+            warn "Tests failed — will attempt self-healing (cycle $((cycle + 1))/$((max_cycles + 1)))"
+            notify "Self-Healing" "Tests failed on cycle ${cycle}, retrying..." "warn"
+        fi
+    done
+
+    error "Self-healing exhausted after $((max_cycles + 1)) cycles"
+    notify "Self-Healing Failed" "Tests still failing after $((max_cycles + 1)) build-test cycles" "error"
+    return 1
+}
+
+# ─── Auto-Rebase Before PR ─────────────────────────────────────────
+auto_rebase() {
+    info "Syncing with ${BASE_BRANCH}..."
+
+    # Fetch latest
+    git fetch origin "$BASE_BRANCH" --quiet 2>/dev/null || {
+        warn "Could not fetch origin/${BASE_BRANCH}"
+        return 0
+    }
+
+    # Check if rebase is needed
+    local behind
+    behind=$(git rev-list --count "HEAD..origin/${BASE_BRANCH}" 2>/dev/null || echo "0")
+
+    if [[ "$behind" -eq 0 ]]; then
+        success "Already up to date with ${BASE_BRANCH}"
+        return 0
+    fi
+
+    info "Rebasing onto origin/${BASE_BRANCH} ($behind commits behind)..."
+    if git rebase "origin/${BASE_BRANCH}" --quiet 2>/dev/null; then
+        success "Rebase successful"
+    else
+        warn "Rebase conflict detected — aborting rebase"
+        git rebase --abort 2>/dev/null || true
+        warn "Falling back to merge..."
+        if git merge "origin/${BASE_BRANCH}" --no-edit --quiet 2>/dev/null; then
+            success "Merge successful"
+        else
+            git merge --abort 2>/dev/null || true
+            error "Both rebase and merge failed — manual intervention needed"
+            return 1
+        fi
+    fi
+}
+
+# ─── Main Pipeline Orchestration ───────────────────────────────────
+run_pipeline() {
+    # Rotate event log if needed (standalone mode)
+    rotate_event_log_if_needed
+
+    # Initialize audit trail for this pipeline run
+    if type audit_init >/dev/null 2>&1; then
+        audit_init || true
+    fi
+
+    local stages
+    stages=$(jq -c '.stages[]' "$PIPELINE_CONFIG" 2>/dev/null)
+
+    local stage_count enabled_count
+    stage_count=$(jq '.stages | length' "$PIPELINE_CONFIG" 2>/dev/null)
+    enabled_count=$(jq '[.stages[] | select(.enabled == true)] | length' "$PIPELINE_CONFIG" 2>/dev/null)
+    local completed=0
+
+    # Check which stages are enabled to determine if we use the self-healing loop
+    local build_enabled test_enabled
+    build_enabled=$(jq -r '.stages[] | select(.id == "build") | .enabled' "$PIPELINE_CONFIG" 2>/dev/null)
+    test_enabled=$(jq -r '.stages[] | select(.id == "test") | .enabled' "$PIPELINE_CONFIG" 2>/dev/null)
+    local use_self_healing=false
+    if [[ "$build_enabled" == "true" && "$test_enabled" == "true" && "$BUILD_TEST_RETRIES" -gt 0 ]]; then
+        use_self_healing=true
+    fi
+
+    while IFS= read -r -u 3 stage; do
+        local id enabled gate
+        id=$(echo "$stage" | jq -r '.id' 2>/dev/null)
+        enabled=$(echo "$stage" | jq -r '.enabled' 2>/dev/null)
+        gate=$(echo "$stage" | jq -r '.gate' 2>/dev/null)
+
+        CURRENT_STAGE_ID="$id"
+
+        # Human intervention: check for skip-stage directive
+        if [[ -f "$ARTIFACTS_DIR/skip-stage.txt" ]]; then
+            local skip_list
+            skip_list="$(cat "$ARTIFACTS_DIR/skip-stage.txt" 2>/dev/null || true)"
+            if echo "$skip_list" | grep -qx "$id" 2>/dev/null; then
+                info "Stage ${BOLD}${id}${RESET} skipped by human directive"
+                emit_event "stage.skipped" "issue=${ISSUE_NUMBER:-0}" "stage=$id" "reason=human_skip"
+                # Remove this stage from the skip file
+                local tmp_skip
+                tmp_skip="$(mktemp)" || { warn "mktemp failed"; continue; }
+                # shellcheck disable=SC2064  # intentional expansion at definition time
+                trap "rm -f '$tmp_skip'" RETURN
+                grep -vx "$id" "$ARTIFACTS_DIR/skip-stage.txt" > "$tmp_skip" 2>/dev/null || true
+                mv "$tmp_skip" "$ARTIFACTS_DIR/skip-stage.txt"
+                continue
+            fi
+        fi
+
+        # Human intervention: check for human message
+        if [[ -f "$ARTIFACTS_DIR/human-message.txt" ]]; then
+            local human_msg
+            human_msg="$(cat "$ARTIFACTS_DIR/human-message.txt" 2>/dev/null || true)"
+            if [[ -n "$human_msg" ]]; then
+                echo ""
+                echo -e "  ${PURPLE}${BOLD}💬 Human message:${RESET} $human_msg"
+                emit_event "pipeline.human_message" "issue=${ISSUE_NUMBER:-0}" "stage=$id" "message=$human_msg"
+                rm -f "$ARTIFACTS_DIR/human-message.txt"
+            fi
+        fi
+
+        if [[ "$enabled" != "true" ]]; then
+            echo -e "  ${DIM}○ ${id} — skipped (disabled)${RESET}"
+            continue
+        fi
+
+        # Intelligence: evaluate whether to skip this stage (after intake, which populates ISSUE_LABELS)
+        if [[ "$id" != "intake" ]] && type pipeline_should_skip_stage >/dev/null 2>&1; then
+            local skip_reason=""
+            skip_reason=$(pipeline_should_skip_stage "$id" 2>/dev/null) || true
+            if [[ -n "$skip_reason" ]]; then
+                echo -e "  ${DIM}○ ${id} — skipped (intelligence: ${skip_reason})${RESET}"
+                set_stage_status "$id" "complete"
+                completed=$((completed + 1))
+                continue
+            fi
+        fi
+
+        local stage_status
+        stage_status=$(get_stage_status "$id")
+        if [[ "$stage_status" == "complete" ]]; then
+            echo -e "  ${GREEN}✓ ${id}${RESET} ${DIM}— already complete${RESET}"
+            completed=$((completed + 1))
+            continue
+        fi
+
+        # CI resume: skip stages marked as completed from previous run
+        if [[ -n "${COMPLETED_STAGES:-}" ]] && echo "$COMPLETED_STAGES" | tr ',' '\n' | grep -qx "$id"; then
+            # Verify artifacts survived the merge — regenerate if missing
+            if verify_stage_artifacts "$id"; then
+                echo -e "  ${GREEN}✓ ${id}${RESET} ${DIM}— skipped (CI resume)${RESET}"
+                set_stage_status "$id" "complete"
+                completed=$((completed + 1))
+                emit_event "stage.skipped" "issue=${ISSUE_NUMBER:-0}" "stage=$id" "reason=ci_resume"
+                continue
+            else
+                warn "Stage $id marked complete but artifacts missing — regenerating"
+                emit_event "stage.artifact_miss" "issue=${ISSUE_NUMBER:-0}" "stage=$id"
+            fi
+        fi
+
+        # Self-healing build→test loop: when we hit build, run both together
+        if [[ "$id" == "build" && "$use_self_healing" == "true" ]]; then
+            # TDD: generate tests before build when enabled
+            if [[ "${TDD_ENABLED:-false}" == "true" || "${PIPELINE_TDD:-}" == "true" ]]; then
+                stage_test_first || true
+            fi
+            # Gate check for build
+            local build_gate
+            build_gate=$(echo "$stage" | jq -r '.gate' 2>/dev/null)
+            if [[ "$build_gate" == "approve" && "$SKIP_GATES" != "true" ]]; then
+                show_stage_preview "build"
+                local answer=""
+                if [[ -t 0 ]]; then
+                    read -rp "  Proceed with build+test (self-healing)? [Y/n] " answer || true
+                fi
+                if [[ "$answer" =~ ^[Nn] ]]; then
+                    update_status "paused" "build"
+                    info "Pipeline paused. Resume with: ${DIM}shipwright pipeline resume${RESET}"
+                    return 0
+                fi
+            fi
+
+            if self_healing_build_test; then
+                completed=$((completed + 2))  # Both build and test
+
+                # Intelligence: reassess complexity after build+test
+                local reassessment
+                reassessment=$(pipeline_reassess_complexity 2>/dev/null) || true
+                if [[ -n "$reassessment" && "$reassessment" != "as_expected" ]]; then
+                    info "Complexity reassessment: ${reassessment}"
+                fi
+            else
+                update_status "failed" "test"
+                error "Pipeline failed: build→test self-healing exhausted"
+                return 1
+            fi
+            continue
+        fi
+
+        # TDD: generate tests before build when enabled (non-self-healing path)
+        if [[ "$id" == "build" && "$use_self_healing" != "true" ]] && [[ "${TDD_ENABLED:-false}" == "true" || "${PIPELINE_TDD:-}" == "true" ]]; then
+            stage_test_first || true
+        fi
+
+        # Skip test if already handled by self-healing loop
+        if [[ "$id" == "test" && "$use_self_healing" == "true" ]]; then
+            stage_status=$(get_stage_status "test")
+            if [[ "$stage_status" == "complete" ]]; then
+                echo -e "  ${GREEN}✓ test${RESET} ${DIM}— completed in build→test loop${RESET}"
+            fi
+            continue
+        fi
+
+        # Gate check
+        if [[ "$gate" == "approve" && "$SKIP_GATES" != "true" ]]; then
+            show_stage_preview "$id"
+            local answer=""
+            if [[ -t 0 ]]; then
+                read -rp "  Proceed with ${id}? [Y/n] " answer || true
+            else
+                # Non-interactive: auto-approve (shouldn't reach here if headless detection works)
+                info "Non-interactive mode — auto-approving ${id}"
+            fi
+            if [[ "$answer" =~ ^[Nn] ]]; then
+                update_status "paused" "$id"
+                info "Pipeline paused at ${BOLD}$id${RESET}. Resume with: ${DIM}shipwright pipeline resume${RESET}"
+                return 0
+            fi
+        fi
+
+        # Budget enforcement check (skip with --ignore-budget)
+        if [[ "$IGNORE_BUDGET" != "true" ]] && [[ -x "$SCRIPT_DIR/sw-cost.sh" ]]; then
+            local budget_rc=0
+            bash "$SCRIPT_DIR/sw-cost.sh" check-budget 2>/dev/null || budget_rc=$?
+            if [[ "$budget_rc" -eq 2 ]]; then
+                warn "Daily budget exceeded — pausing pipeline before stage ${BOLD}$id${RESET}"
+                warn "Resume with --ignore-budget to override, or wait until tomorrow"
+                emit_event "pipeline.budget_paused" "issue=${ISSUE_NUMBER:-0}" "stage=$id"
+                update_status "paused" "$id"
+                return 0
+            fi
+        fi
+
+        # Intelligence: per-stage model routing (UCB1 when DB has data, else A/B testing)
+        local recommended_model="" from_ucb1=false
+        if type ucb1_select_model >/dev/null 2>&1; then
+            recommended_model=$(ucb1_select_model "$id" 2>/dev/null || echo "")
+            [[ -n "$recommended_model" ]] && from_ucb1=true
+        fi
+        if [[ -z "$recommended_model" ]] && type intelligence_recommend_model >/dev/null 2>&1; then
+            local stage_complexity="${INTELLIGENCE_COMPLEXITY:-5}"
+            local budget_remaining=""
+            if [[ -x "$SCRIPT_DIR/sw-cost.sh" ]]; then
+                budget_remaining=$(bash "$SCRIPT_DIR/sw-cost.sh" remaining-budget 2>/dev/null || echo "")
+            fi
+            local recommended_json
+            recommended_json=$(intelligence_recommend_model "$id" "$stage_complexity" "$budget_remaining" 2>/dev/null || echo "")
+            recommended_model=$(echo "$recommended_json" | jq -r '.model // empty' 2>/dev/null || echo "")
+        fi
+        if [[ -n "$recommended_model" && "$recommended_model" != "null" ]]; then
+            if [[ "$from_ucb1" == "true" ]]; then
+                # UCB1 already balances exploration/exploitation — use directly
+                export CLAUDE_MODEL="$recommended_model"
+                emit_event "intelligence.model_ucb1" \
+                    "issue=${ISSUE_NUMBER:-0}" \
+                    "stage=$id" \
+                    "model=$recommended_model"
+            else
+                # A/B testing for intelligence recommendation
+                local ab_ratio=20
+                local daemon_cfg="${PROJECT_ROOT}/.claude/daemon-config.json"
+                if [[ -f "$daemon_cfg" ]]; then
+                    local cfg_ratio
+                    cfg_ratio=$(jq -r '.intelligence.ab_test_ratio // 0.2' "$daemon_cfg" 2>/dev/null || echo "0.2")
+                    ab_ratio=$(awk -v r="$cfg_ratio" 'BEGIN{printf "%d", r * 100}' 2>/dev/null || echo "20")
+                fi
+
+                local routing_file="${HOME}/.shipwright/optimization/model-routing.json"
+                local use_recommended=false
+                local ab_group="control"
+
+                if [[ -f "$routing_file" ]]; then
+                    local stage_samples total_samples
+                    stage_samples=$(jq -r --arg s "$id" '.routes[$s].sonnet_samples // .[$s].sonnet_samples // 0' "$routing_file" 2>/dev/null || echo "0")
+                    total_samples=$(jq -r --arg s "$id" '((.routes[$s].sonnet_samples // .[$s].sonnet_samples // 0) + (.routes[$s].opus_samples // .[$s].opus_samples // 0))' "$routing_file" 2>/dev/null || echo "0")
+                    if [[ "${total_samples:-0}" -ge 50 ]]; then
+                        use_recommended=true
+                        ab_group="graduated"
+                    fi
+                fi
+
+                if [[ "$use_recommended" != "true" ]]; then
+                    local roll=$((RANDOM % 100))
+                    if [[ "$roll" -lt "$ab_ratio" ]]; then
+                        use_recommended=true
+                        ab_group="experiment"
+                    fi
+                fi
+
+                if [[ "$use_recommended" == "true" ]]; then
+                    export CLAUDE_MODEL="$recommended_model"
+                else
+                    export CLAUDE_MODEL="$(_smart_model default sonnet)"
+                fi
+
+                emit_event "intelligence.model_ab" \
+                    "issue=${ISSUE_NUMBER:-0}" \
+                    "stage=$id" \
+                    "recommended=$recommended_model" \
+                    "applied=$CLAUDE_MODEL" \
+                    "ab_group=$ab_group" \
+                    "ab_ratio=$ab_ratio"
+            fi
+        fi
+
+        echo ""
+        echo -e "${CYAN}${BOLD}▸ Stage: ${id}${RESET} ${DIM}[$((completed + 1))/${enabled_count}]${RESET}"
+        update_status "running" "$id"
+        record_stage_start "$id"
+        local stage_start_epoch
+        stage_start_epoch=$(now_epoch)
+        emit_event "stage.started" "issue=${ISSUE_NUMBER:-0}" "stage=$id"
+
+        # Mark GitHub Check Run as in-progress
+        if [[ "${NO_GITHUB:-false}" != "true" ]] && type gh_checks_stage_update >/dev/null 2>&1; then
+            gh_checks_stage_update "$id" "in_progress" "" "Stage $id started" 2>/dev/null || true
+        fi
+
+        # Audit: stage start
+        if type audit_emit >/dev/null 2>&1; then
+            audit_emit "stage.start" "stage=$id" || true
+        fi
+
+        local stage_model_used="${CLAUDE_MODEL:-${MODEL:-opus}}"
+        if run_stage_with_retry "$id"; then
+            mark_stage_complete "$id"
+            completed=$((completed + 1))
+            # Capture project pattern after intake (for memory context in later stages)
+            if [[ "$id" == "intake" ]] && [[ -x "$SCRIPT_DIR/sw-memory.sh" ]]; then
+                (cd "$REPO_DIR" && bash "$SCRIPT_DIR/sw-memory.sh" pattern "project" "{}" 2>/dev/null) || true
+            fi
+            local timing stage_dur_s
+            timing=$(get_stage_timing "$id")
+            stage_dur_s=$(( $(now_epoch) - stage_start_epoch ))
+            success "Stage ${BOLD}$id${RESET} complete ${DIM}(${timing})${RESET}"
+            emit_event "stage.completed" "issue=${ISSUE_NUMBER:-0}" "stage=$id" "duration_s=$stage_dur_s" "result=success"
+            # Audit: stage complete
+            if type audit_emit >/dev/null 2>&1; then
+                audit_emit "stage.complete" "stage=$id" "verdict=pass" \
+                    "duration_s=${stage_dur_s:-0}" || true
+            fi
+            # Emit vitals snapshot on every stage transition (not just build/test)
+            if type pipeline_emit_progress_snapshot >/dev/null 2>&1 && [[ -n "${ISSUE_NUMBER:-}" ]]; then
+                pipeline_emit_progress_snapshot "${ISSUE_NUMBER}" "$id" "0" "0" "0" "" 2>/dev/null || true
+            fi
+            # Record model outcome for UCB1 learning
+            type record_model_outcome >/dev/null 2>&1 && record_model_outcome "$stage_model_used" "$id" 1 "$stage_dur_s" 0 2>/dev/null || true
+            # Broadcast discovery for cross-pipeline learning
+            if [[ -x "$SCRIPT_DIR/sw-discovery.sh" ]]; then
+                local _disc_cat _disc_patterns _disc_text
+                _disc_cat="$id"
+                case "$id" in
+                    plan)   _disc_patterns="*.md"; _disc_text="Plan completed: ${GOAL:-goal}" ;;
+                    design) _disc_patterns="*.md,*.ts,*.tsx,*.js"; _disc_text="Design completed for ${GOAL:-goal}" ;;
+                    build)  _disc_patterns="src/*,*.ts,*.tsx,*.js"; _disc_text="Build completed" ;;
+                    test)   _disc_patterns="*.test.*,*_test.*"; _disc_text="Tests passed" ;;
+                    review) _disc_patterns="*.md,*.ts,*.tsx"; _disc_text="Review completed" ;;
+                    *)      _disc_patterns="*"; _disc_text="Stage $id completed" ;;
+                esac
+                bash "$SCRIPT_DIR/sw-discovery.sh" broadcast "$_disc_cat" "$_disc_patterns" "$_disc_text" "" 2>/dev/null || true
+            fi
+            # Log model used for prediction feedback
+            echo "${id}|${stage_model_used}|true" >> "${ARTIFACTS_DIR}/model-routing.log"
+        else
+            mark_stage_failed "$id"
+            local stage_dur_s
+            stage_dur_s=$(( $(now_epoch) - stage_start_epoch ))
+            error "Pipeline failed at stage: ${BOLD}$id${RESET}"
+            update_status "failed" "$id"
+            emit_event "stage.failed" \
+                "issue=${ISSUE_NUMBER:-0}" \
+                "stage=$id" \
+                "duration_s=$stage_dur_s" \
+                "error=${LAST_STAGE_ERROR:-unknown}" \
+                "error_class=${LAST_STAGE_ERROR_CLASS:-unknown}"
+            # Audit: stage failed
+            if type audit_emit >/dev/null 2>&1; then
+                audit_emit "stage.complete" "stage=$id" "verdict=fail" \
+                    "duration_s=${stage_dur_s:-0}" || true
+            fi
+            # Emit vitals snapshot on failure too
+            if type pipeline_emit_progress_snapshot >/dev/null 2>&1 && [[ -n "${ISSUE_NUMBER:-}" ]]; then
+                pipeline_emit_progress_snapshot "${ISSUE_NUMBER}" "$id" "0" "0" "0" "${LAST_STAGE_ERROR:-unknown}" 2>/dev/null || true
+            fi
+            # Log model used for prediction feedback
+            echo "${id}|${stage_model_used}|false" >> "${ARTIFACTS_DIR}/model-routing.log"
+            # Record model outcome for UCB1 learning
+            type record_model_outcome >/dev/null 2>&1 && record_model_outcome "$stage_model_used" "$id" 0 "$stage_dur_s" 0 2>/dev/null || true
+            # Cancel any remaining in_progress check runs
+            pipeline_cancel_check_runs 2>/dev/null || true
+            return 1
+        fi
+    done 3<<< "$stages"
+
+    # Pipeline complete!
+    update_status "complete" ""
+    PIPELINE_STAGES_PASSED="$completed"
+    PIPELINE_SLOWEST_STAGE=""
+    if type get_slowest_stage >/dev/null 2>&1; then
+        PIPELINE_SLOWEST_STAGE=$(get_slowest_stage 2>/dev/null || true)
+    fi
+    local total_dur=""
+    if [[ -n "$PIPELINE_START_EPOCH" ]]; then
+        total_dur=$(format_duration $(( $(now_epoch) - PIPELINE_START_EPOCH )))
+    fi
+
+    echo ""
+    echo -e "${GREEN}${BOLD}═══════════════════════════════════════════════════════════════════${RESET}"
+    success "Pipeline complete! ${completed}/${enabled_count} stages passed in ${total_dur:-unknown}"
+    echo -e "${GREEN}${BOLD}═══════════════════════════════════════════════════════════════════${RESET}"
+
+    # Show summary
+    echo ""
+    if [[ -f "$ARTIFACTS_DIR/pr-url.txt" ]]; then
+        echo -e "  ${BOLD}PR:${RESET}        $(cat "$ARTIFACTS_DIR/pr-url.txt")"
+    fi
+    echo -e "  ${BOLD}Branch:${RESET}    $GIT_BRANCH"
+    [[ -n "${GITHUB_ISSUE:-}" ]] && echo -e "  ${BOLD}Issue:${RESET}     $GITHUB_ISSUE"
+    echo -e "  ${BOLD}Duration:${RESET}  $total_dur"
+    echo -e "  ${BOLD}Artifacts:${RESET} $ARTIFACTS_DIR/"
+    echo ""
+
+    # Capture learnings to memory (success or failure)
+    if [[ -x "$SCRIPT_DIR/sw-memory.sh" ]]; then
+        bash "$SCRIPT_DIR/sw-memory.sh" capture "$STATE_FILE" "$ARTIFACTS_DIR" 2>/dev/null || true
+    fi
+
+    # Final GitHub progress update
+    if [[ -n "$ISSUE_NUMBER" ]]; then
+        local body
+        body=$(gh_build_progress_body)
+        gh_update_progress "$body"
+    fi
+
+    # Post-completion cleanup
+    pipeline_post_completion_cleanup
+}