shipwright-cli 3.2.0 → 3.3.0

package/scripts/lib/pipeline-stages-monitor.sh

@@ -0,0 +1,407 @@
+# pipeline-stages-monitor.sh — validate, monitor stages
+# Source from pipeline-stages.sh. Requires all pipeline globals and dependencies.
+[[ -n "${_PIPELINE_STAGES_MONITOR_LOADED:-}" ]] && return 0
+_PIPELINE_STAGES_MONITOR_LOADED=1
+
+stage_validate() {
+    CURRENT_STAGE_ID="validate"
+    # Consume retry context if this is a retry attempt
+    local _retry_ctx="${ARTIFACTS_DIR}/.retry-context-validate.md"
+    if [[ -s "$_retry_ctx" ]]; then
+        local _validate_retry_hints
+        _validate_retry_hints=$(cat "$_retry_ctx" 2>/dev/null || true)
+        rm -f "$_retry_ctx"
+    fi
+    # Load validation thoroughness skills
+    if type skill_load_prompts >/dev/null 2>&1; then
+        local _validate_skills
+        _validate_skills=$(skill_load_prompts "${INTELLIGENCE_ISSUE_TYPE:-backend}" "validate" 2>/dev/null || true)
+        if [[ -n "$_validate_skills" ]]; then
+            echo "$_validate_skills" > "${ARTIFACTS_DIR}/.validation-skills.md" 2>/dev/null || true
+        fi
+    fi
+    local smoke_cmd
+    smoke_cmd=$(jq -r --arg id "validate" '(.stages[] | select(.id == $id) | .config.smoke_cmd) // ""' "$PIPELINE_CONFIG" 2>/dev/null) || true
+    [[ "$smoke_cmd" == "null" ]] && smoke_cmd=""
+
+    local health_url
+    health_url=$(jq -r --arg id "validate" '(.stages[] | select(.id == $id) | .config.health_url) // ""' "$PIPELINE_CONFIG" 2>/dev/null) || true
+    [[ "$health_url" == "null" ]] && health_url=""
+
+    local close_issue
+    close_issue=$(jq -r --arg id "validate" '(.stages[] | select(.id == $id) | .config.close_issue) // false' "$PIPELINE_CONFIG" 2>/dev/null) || true
+
+    # Smoke tests
+    if [[ -n "$smoke_cmd" ]]; then
+        info "Running smoke tests..."
+        bash -c "$smoke_cmd" > "$ARTIFACTS_DIR/smoke.log" 2>&1 || {
+            error "Smoke tests failed"
+            if [[ -n "$ISSUE_NUMBER" ]]; then
+                gh issue create --title "Deploy validation failed: $GOAL" \
+                    --label "incident" --body "Pipeline smoke tests failed after deploy.
+
+Related issue: ${GITHUB_ISSUE}
+Branch: ${GIT_BRANCH}
+PR: $(cat "$ARTIFACTS_DIR/pr-url.txt" 2>/dev/null || echo 'unknown')" 2>/dev/null || true
+            fi
+            return 1
+        }
+        success "Smoke tests passed"
+    fi
+
+    # Health check with retry
+    if [[ -n "$health_url" ]]; then
+        info "Health check: $health_url"
+        local attempts=0
+        while [[ $attempts -lt 5 ]]; do
+            if curl -sf "$health_url" >/dev/null 2>&1; then
+                success "Health check passed"
+                break
+            fi
+            attempts=$((attempts + 1))
+            [[ $attempts -lt 5 ]] && { info "Retry ${attempts}/5..."; sleep "$(_exponential_backoff "$attempts" 5 60)"; }
+        done
+        if [[ $attempts -ge 5 ]]; then
+            error "Health check failed after 5 attempts"
+            return 1
+        fi
+    fi
+
+    # Compute total duration once for both issue close and wiki report
+    local total_dur=""
+    if [[ -n "$PIPELINE_START_EPOCH" ]]; then
+        total_dur=$(format_duration $(( $(now_epoch) - PIPELINE_START_EPOCH )))
+    fi
+
+    # Close original issue with comprehensive summary
+    if [[ "$close_issue" == "true" && -n "$ISSUE_NUMBER" ]]; then
+        gh issue close "$ISSUE_NUMBER" --comment "## ✅ Complete — Deployed & Validated
+
+| Metric | Value |
+|--------|-------|
+| Pipeline | \`${PIPELINE_NAME}\` |
+| Branch | \`${GIT_BRANCH}\` |
+| PR | $(cat "$ARTIFACTS_DIR/pr-url.txt" 2>/dev/null || echo 'N/A') |
+| Duration | ${total_dur:-unknown} |
+
+_Closed automatically by \`shipwright pipeline\`_" 2>/dev/null || true
+
+        gh_remove_label "$ISSUE_NUMBER" "pipeline/pr-created"
+        gh_add_labels "$ISSUE_NUMBER" "pipeline/complete"
+        success "Issue #$ISSUE_NUMBER closed"
+    fi
+
+    # Push pipeline report to wiki
+    local report="# Pipeline Report — ${GOAL}
+
+| Metric | Value |
+|--------|-------|
+| Pipeline | \`${PIPELINE_NAME}\` |
+| Branch | \`${GIT_BRANCH}\` |
+| PR | $(cat "$ARTIFACTS_DIR/pr-url.txt" 2>/dev/null || echo 'N/A') |
+| Duration | ${total_dur:-unknown} |
+| Stages | $(echo "$STAGE_TIMINGS" | tr '|' '\n' | wc -l | xargs) completed |
+
+## Stage Timings
+$(echo "$STAGE_TIMINGS" | tr '|' '\n' | sed 's/^/- /')
+
+## Artifacts
+$(ls -1 "$ARTIFACTS_DIR" 2>/dev/null | sed 's/^/- /')
+
+---
+_Generated by \`shipwright pipeline\` at $(now_iso)_"
+    gh_wiki_page "Pipeline-Report-${ISSUE_NUMBER:-inline}" "$report"
+
+    log_stage "validate" "Validation complete"
+}
+
+stage_monitor() {
+    CURRENT_STAGE_ID="monitor"
+    # Consume retry context if this is a retry attempt
+    local _retry_ctx="${ARTIFACTS_DIR}/.retry-context-monitor.md"
+    if [[ -s "$_retry_ctx" ]]; then
+        local _monitor_retry_hints
+        _monitor_retry_hints=$(cat "$_retry_ctx" 2>/dev/null || true)
+        rm -f "$_retry_ctx"
+    fi
+    # Load observability skills
+    if type skill_load_prompts >/dev/null 2>&1; then
+        local _monitor_skills
+        _monitor_skills=$(skill_load_prompts "${INTELLIGENCE_ISSUE_TYPE:-backend}" "monitor" 2>/dev/null || true)
+        if [[ -n "$_monitor_skills" ]]; then
+            echo "$_monitor_skills" > "${ARTIFACTS_DIR}/.monitoring-skills.md" 2>/dev/null || true
+        fi
+    fi
+
+    # Read config from pipeline template
+    local duration_minutes health_url error_threshold log_pattern log_cmd rollback_cmd auto_rollback
+    duration_minutes=$(jq -r --arg id "monitor" '(.stages[] | select(.id == $id) | .config.duration_minutes) // 5' "$PIPELINE_CONFIG" 2>/dev/null) || true
+    [[ -z "$duration_minutes" || "$duration_minutes" == "null" ]] && duration_minutes=5
+    health_url=$(jq -r --arg id "monitor" '(.stages[] | select(.id == $id) | .config.health_url) // ""' "$PIPELINE_CONFIG" 2>/dev/null) || true
+    [[ "$health_url" == "null" ]] && health_url=""
+    error_threshold=$(jq -r --arg id "monitor" '(.stages[] | select(.id == $id) | .config.error_threshold) // 5' "$PIPELINE_CONFIG" 2>/dev/null) || true
+    [[ -z "$error_threshold" || "$error_threshold" == "null" ]] && error_threshold=5
+
+    # Adaptive monitor: use historical baselines if available
+    local repo_hash
+    repo_hash=$(echo "${PROJECT_ROOT:-$(pwd)}" | cksum | awk '{print $1}')
+    local baseline_file="${HOME}/.shipwright/baselines/${repo_hash}/deploy-monitor.json"
+    if [[ -f "$baseline_file" ]]; then
+        local hist_duration hist_threshold
+        hist_duration=$(jq -r '.p90_stabilization_minutes // empty' "$baseline_file" 2>/dev/null || true)
+        hist_threshold=$(jq -r '.p90_error_threshold // empty' "$baseline_file" 2>/dev/null || true)
+        if [[ -n "$hist_duration" && "$hist_duration" != "null" ]]; then
+            duration_minutes="$hist_duration"
+            info "Monitor duration: ${duration_minutes}m ${DIM}(from baseline)${RESET}"
+        fi
+        if [[ -n "$hist_threshold" && "$hist_threshold" != "null" ]]; then
+            error_threshold="$hist_threshold"
+            info "Error threshold: ${error_threshold} ${DIM}(from baseline)${RESET}"
+        fi
+    fi
+    log_pattern=$(jq -r --arg id "monitor" '(.stages[] | select(.id == $id) | .config.log_pattern) // "ERROR|FATAL|PANIC"' "$PIPELINE_CONFIG" 2>/dev/null) || true
+    [[ -z "$log_pattern" || "$log_pattern" == "null" ]] && log_pattern="ERROR|FATAL|PANIC"
+    log_cmd=$(jq -r --arg id "monitor" '(.stages[] | select(.id == $id) | .config.log_cmd) // ""' "$PIPELINE_CONFIG" 2>/dev/null) || true
+    [[ "$log_cmd" == "null" ]] && log_cmd=""
+    rollback_cmd=$(jq -r --arg id "monitor" '(.stages[] | select(.id == $id) | .config.rollback_cmd) // ""' "$PIPELINE_CONFIG" 2>/dev/null) || true
+    [[ "$rollback_cmd" == "null" ]] && rollback_cmd=""
+    auto_rollback=$(jq -r --arg id "monitor" '(.stages[] | select(.id == $id) | .config.auto_rollback) // false' "$PIPELINE_CONFIG" 2>/dev/null) || true
+    [[ -z "$auto_rollback" || "$auto_rollback" == "null" ]] && auto_rollback="false"
+
+    if [[ -z "$health_url" && -z "$log_cmd" ]]; then
+        warn "No health_url or log_cmd configured — skipping monitor stage"
+        log_stage "monitor" "Skipped (no monitoring configured)"
+        return 0
+    fi
+
+    local report_file="$ARTIFACTS_DIR/monitor-report.md"
+    local deploy_log_file="$ARTIFACTS_DIR/deploy-logs.txt"
+    : > "$deploy_log_file"
+    local total_errors=0
+    local poll_interval=30  # seconds between polls
+    local total_polls=$(( (duration_minutes * 60) / poll_interval ))
+    [[ "$total_polls" -lt 1 ]] && total_polls=1
+
+    info "Post-deploy monitoring: ${duration_minutes}m (${total_polls} polls, threshold: ${error_threshold} errors)"
+
+    emit_event "monitor.started" \
+        "issue=${ISSUE_NUMBER:-0}" \
+        "duration_minutes=$duration_minutes" \
+        "error_threshold=$error_threshold"
+
+    {
+        echo "# Post-Deploy Monitor Report"
+        echo ""
+        echo "- Duration: ${duration_minutes} minutes"
+        echo "- Health URL: ${health_url:-none}"
+        echo "- Log command: ${log_cmd:-none}"
+        echo "- Error threshold: ${error_threshold}"
+        echo "- Auto-rollback: ${auto_rollback}"
+        echo ""
+        echo "## Poll Results"
+        echo ""
+    } > "$report_file"
+
+    local poll=0
+    local health_failures=0
+    local log_errors=0
+    while [[ "$poll" -lt "$total_polls" ]]; do
+        poll=$((poll + 1))
+        local poll_time
+        poll_time=$(now_iso)
+
+        # Health URL check
+        if [[ -n "$health_url" ]]; then
+            local http_status
+            http_status=$(curl -sf -o /dev/null -w "%{http_code}" "$health_url" 2>/dev/null || echo "000")
+            if [[ "$http_status" -ge 200 && "$http_status" -lt 400 ]]; then
+                echo "- [${poll_time}] Health: ✅ (HTTP ${http_status})" >> "$report_file"
+            else
+                health_failures=$((health_failures + 1))
+                total_errors=$((total_errors + 1))
+                echo "- [${poll_time}] Health: ❌ (HTTP ${http_status})" >> "$report_file"
+                warn "Health check failed: HTTP ${http_status}"
+            fi
+        fi
+
+        # Log command check (accumulate deploy logs for feedback collect)
+        if [[ -n "$log_cmd" ]]; then
+            local log_output
+            log_output=$(bash -c "$log_cmd" 2>/dev/null || true)
+            [[ -n "$log_output" ]] && echo "$log_output" >> "$deploy_log_file"
+            local error_count=0
+            if [[ -n "$log_output" ]]; then
+                error_count=$(echo "$log_output" | grep -cE "$log_pattern" 2>/dev/null || true)
+                error_count="${error_count:-0}"
+            fi
+            if [[ "$error_count" -gt 0 ]]; then
+                log_errors=$((log_errors + error_count))
+                total_errors=$((total_errors + error_count))
+                echo "- [${poll_time}] Logs: ⚠️ ${error_count} error(s) matching '${log_pattern}'" >> "$report_file"
+                warn "Log errors detected: ${error_count}"
+            else
+                echo "- [${poll_time}] Logs: ✅ clean" >> "$report_file"
+            fi
+        fi
+
+        emit_event "monitor.check" \
+            "issue=${ISSUE_NUMBER:-0}" \
+            "poll=$poll" \
+            "total_errors=$total_errors" \
+            "health_failures=$health_failures"
+
+        # Check threshold
+        if [[ "$total_errors" -ge "$error_threshold" ]]; then
+            error "Error threshold exceeded: ${total_errors} >= ${error_threshold}"
+
+            echo "" >> "$report_file"
+            echo "## ❌ THRESHOLD EXCEEDED" >> "$report_file"
+            echo "Total errors: ${total_errors} (threshold: ${error_threshold})" >> "$report_file"
+
+            emit_event "monitor.alert" \
+                "issue=${ISSUE_NUMBER:-0}" \
+                "total_errors=$total_errors" \
+                "threshold=$error_threshold"
+
+            # Feedback loop: collect deploy logs and optionally create issue
+            if [[ -f "$deploy_log_file" ]] && [[ -s "$deploy_log_file" ]] && [[ -x "$SCRIPT_DIR/sw-feedback.sh" ]]; then
+                (cd "$PROJECT_ROOT" && ARTIFACTS_DIR="$ARTIFACTS_DIR" bash "$SCRIPT_DIR/sw-feedback.sh" collect "$deploy_log_file" 2>/dev/null) || true
+                (cd "$PROJECT_ROOT" && ARTIFACTS_DIR="$ARTIFACTS_DIR" bash "$SCRIPT_DIR/sw-feedback.sh" create-issue 2>/dev/null) || true
+            fi
+
+            # Auto-rollback: feedback rollback (GitHub Deployments API) and/or config rollback_cmd
+            if [[ "$auto_rollback" == "true" ]]; then
+                warn "Auto-rolling back..."
+                echo "" >> "$report_file"
+                echo "## Rollback" >> "$report_file"
+
+                # Trigger feedback rollback (calls sw-github-deploy.sh rollback)
+                if [[ -x "$SCRIPT_DIR/sw-feedback.sh" ]]; then
+                    (cd "$PROJECT_ROOT" && ARTIFACTS_DIR="$ARTIFACTS_DIR" bash "$SCRIPT_DIR/sw-feedback.sh" rollback production "Monitor threshold exceeded (${total_errors} errors)" >> "$report_file" 2>&1) || true
+                fi
+
+                if [[ -n "$rollback_cmd" ]] && bash -c "$rollback_cmd" >> "$report_file" 2>&1; then
+                    success "Rollback executed"
+                    echo "Rollback: ✅ success" >> "$report_file"
+
+                    # Post-rollback smoke test verification
+                    local smoke_cmd
+                    smoke_cmd=$(jq -r --arg id "validate" '(.stages[] | select(.id == $id) | .config.smoke_cmd) // ""' "$PIPELINE_CONFIG" 2>/dev/null) || true
+                    [[ "$smoke_cmd" == "null" ]] && smoke_cmd=""
+
+                    if [[ -n "$smoke_cmd" ]]; then
+                        info "Verifying rollback with smoke tests..."
+                        if bash -c "$smoke_cmd" > "$ARTIFACTS_DIR/rollback-smoke.log" 2>&1; then
+                            success "Rollback verified — smoke tests pass"
+                            echo "Rollback verification: ✅ smoke tests pass" >> "$report_file"
+                            emit_event "monitor.rollback_verified" \
+                                "issue=${ISSUE_NUMBER:-0}" \
+                                "status=pass"
+                        else
+                            error "Rollback verification FAILED — smoke tests still failing"
+                            echo "Rollback verification: ❌ smoke tests FAILED — manual intervention required" >> "$report_file"
+                            emit_event "monitor.rollback_verified" \
+                                "issue=${ISSUE_NUMBER:-0}" \
+                                "status=fail"
+                            if [[ -n "$ISSUE_NUMBER" ]]; then
+                                gh_comment_issue "$ISSUE_NUMBER" "🚨 **Rollback executed but verification failed** — smoke tests still failing after rollback. Manual intervention required.
+
+Smoke command: \`${smoke_cmd}\`
+Log: see \`pipeline-artifacts/rollback-smoke.log\`" 2>/dev/null || true
+                            fi
+                        fi
+                    fi
+                else
+                    error "Rollback failed!"
+                    echo "Rollback: ❌ failed" >> "$report_file"
+                fi
+
+                emit_event "monitor.rollback" \
+                    "issue=${ISSUE_NUMBER:-0}" \
+                    "total_errors=$total_errors"
+
+                # Post to GitHub
+                if [[ -n "$ISSUE_NUMBER" ]]; then
+                    gh_comment_issue "$ISSUE_NUMBER" "🚨 **Auto-rollback triggered** — ${total_errors} errors exceeded threshold (${error_threshold})
+
+Rollback command: \`${rollback_cmd}\`" 2>/dev/null || true
+
+                    # Create hotfix issue
+                    if [[ "$GH_AVAILABLE" == "true" ]]; then
+                        gh issue create \
+                            --title "Hotfix: Deploy regression for ${GOAL}" \
+                            --label "hotfix,incident" \
+                            --body "Auto-rollback triggered during post-deploy monitoring.
+
+**Original issue:** ${GITHUB_ISSUE:-N/A}
+**Errors detected:** ${total_errors}
+**Threshold:** ${error_threshold}
+**Branch:** ${GIT_BRANCH}
+
+## Monitor Report
+$(cat "$report_file")
+
+---
+_Created automatically by \`shipwright pipeline\` monitor stage_" 2>/dev/null || true
+                    fi
+                fi
+            fi
+
+            log_stage "monitor" "Failed — ${total_errors} errors (threshold: ${error_threshold})"
+            return 1
+        fi
+
+        # Sleep between polls (skip on last poll)
+        if [[ "$poll" -lt "$total_polls" ]]; then
+            sleep "$poll_interval"
+        fi
+    done
+
+    # Monitoring complete — all clear
+    echo "" >> "$report_file"
+    echo "## ✅ Monitoring Complete" >> "$report_file"
+    echo "Total errors: ${total_errors} (threshold: ${error_threshold})" >> "$report_file"
+    echo "Health failures: ${health_failures}" >> "$report_file"
+    echo "Log errors: ${log_errors}" >> "$report_file"
+
+    success "Post-deploy monitoring clean (${total_errors} errors in ${duration_minutes}m)"
+
+    # Proactive feedback collection: always collect deploy logs for trend analysis
+    if [[ -f "$deploy_log_file" ]] && [[ -s "$deploy_log_file" ]] && [[ -x "$SCRIPT_DIR/sw-feedback.sh" ]]; then
+        (cd "$PROJECT_ROOT" && ARTIFACTS_DIR="$ARTIFACTS_DIR" bash "$SCRIPT_DIR/sw-feedback.sh" collect "$deploy_log_file" 2>/dev/null) || true
+    fi
+
+    if [[ -n "$ISSUE_NUMBER" ]]; then
+        gh_comment_issue "$ISSUE_NUMBER" "✅ **Post-deploy monitoring passed** — ${duration_minutes}m, ${total_errors} errors" 2>/dev/null || true
+    fi
+
+    log_stage "monitor" "Clean — ${total_errors} errors in ${duration_minutes}m"
+
+    # Record baseline for adaptive monitoring on future runs
+    local baseline_dir="${HOME}/.shipwright/baselines/${repo_hash}"
+    mkdir -p "$baseline_dir" 2>/dev/null || true
+    local baseline_tmp
+    baseline_tmp="$(mktemp)"
+    if [[ -f "${baseline_dir}/deploy-monitor.json" ]]; then
+        # Append to history and recalculate p90
+        jq --arg dur "$duration_minutes" --arg errs "$total_errors" \
+            '.history += [{"duration_minutes": ($dur | tonumber), "errors": ($errs | tonumber)}] |
+             .p90_stabilization_minutes = ([.history[].duration_minutes] | sort | .[length * 9 / 10 | floor]) |
+             .p90_error_threshold = (([.history[].errors] | sort | .[length * 9 / 10 | floor]) + 2) |
+             .updated_at = now' \
+            "${baseline_dir}/deploy-monitor.json" > "$baseline_tmp" 2>/dev/null && \
+            mv "$baseline_tmp" "${baseline_dir}/deploy-monitor.json" || rm -f "$baseline_tmp"
+    else
+        jq -n --arg dur "$duration_minutes" --arg errs "$total_errors" \
+            '{history: [{"duration_minutes": ($dur | tonumber), "errors": ($errs | tonumber)}],
+              p90_stabilization_minutes: ($dur | tonumber),
+              p90_error_threshold: (($errs | tonumber) + 2),
+              updated_at: now}' \
+            > "$baseline_tmp" 2>/dev/null && \
+            mv "$baseline_tmp" "${baseline_dir}/deploy-monitor.json" || rm -f "$baseline_tmp"
+    fi
+}
+
+# ─── Multi-Dimensional Quality Checks ─────────────────────────────────────
+# Beyond tests: security, bundle size, perf regression, API compat, coverage
+