shipwright-cli 2.2.0 → 2.2.2

package/scripts/lib/daemon-dispatch.sh

@@ -0,0 +1,489 @@
+# daemon-dispatch.sh — Spawn, reap, on_success (for sw-daemon.sh)
+# Source from sw-daemon.sh. Requires state, failure, helpers.
+[[ -n "${_DAEMON_DISPATCH_LOADED:-}" ]] && return 0
+_DAEMON_DISPATCH_LOADED=1
+
+# ─── Org-Wide Repo Management ─────────────────────────────────────────────
+
+daemon_ensure_repo() {
+    local owner="$1" repo="$2"
+    local repo_dir="$DAEMON_DIR/repos/${owner}/${repo}"
+
+    if [[ -d "$repo_dir/.git" ]]; then
+        # Pull latest
+        (cd "$repo_dir" && git pull --ff-only 2>/dev/null) || {
+            daemon_log WARN "Failed to update ${owner}/${repo} — using existing clone"
+        }
+    else
+        mkdir -p "$DAEMON_DIR/repos/${owner}"
+        if ! git clone --depth=1 "https://github.com/${owner}/${repo}.git" "$repo_dir" 2>/dev/null; then
+            daemon_log ERROR "Failed to clone ${owner}/${repo}"
+            return 1
+        fi
+        daemon_log INFO "Cloned ${owner}/${repo} to ${repo_dir}"
+    fi
+
+    echo "$repo_dir"
+}
+
+# ─── Spawn Pipeline ─────────────────────────────────────────────────────────
+
+daemon_spawn_pipeline() {
+    local issue_num="$1"
+    local issue_title="${2:-}"
+    local repo_full_name="${3:-}"  # owner/repo (org mode only)
+    shift 3 2>/dev/null || true
+    local extra_pipeline_args=("$@")  # Optional extra args passed to sw-pipeline.sh
+
+    daemon_log INFO "Spawning pipeline for issue #${issue_num}: ${issue_title}"
+
+    # ── Issue decomposition (if decomposer available) ──
+    local decompose_script="${SCRIPT_DIR}/sw-decompose.sh"
+    if [[ -x "$decompose_script" && "$NO_GITHUB" != "true" ]]; then
+        local decompose_result=""
+        decompose_result=$("$decompose_script" auto "$issue_num" 2>/dev/null) || true
+        if [[ "$decompose_result" == *"decomposed"* ]]; then
+            daemon_log INFO "Issue #${issue_num} decomposed into subtasks — skipping pipeline"
+            # Remove the shipwright label so decomposed parent doesn't re-queue
+            gh issue edit "$issue_num" --remove-label "shipwright" 2>/dev/null || true
+            return 0
+        fi
+    fi
+
+    # Extract goal text from issue (title + first line of body)
+    local issue_goal="$issue_title"
+    if [[ "$NO_GITHUB" != "true" ]]; then
+        local issue_body_first
+        issue_body_first=$(gh issue view "$issue_num" --json body --jq '.body' 2>/dev/null | head -3 | tr '\n' ' ' | cut -c1-200 || true)
+        if [[ -n "$issue_body_first" ]]; then
+            issue_goal="${issue_title}: ${issue_body_first}"
+        fi
+    fi
+
+    # ── Predictive risk assessment (if enabled) ──
+    if [[ "${PREDICTION_ENABLED:-false}" == "true" ]] && type predict_pipeline_risk &>/dev/null 2>&1; then
+        local issue_json_for_pred=""
+        if [[ "$NO_GITHUB" != "true" ]]; then
+            issue_json_for_pred=$(gh issue view "$issue_num" --json number,title,body,labels 2>/dev/null || echo "")
+        fi
+        if [[ -n "$issue_json_for_pred" ]]; then
+            local risk_result
+            risk_result=$(predict_pipeline_risk "$issue_json_for_pred" "" 2>/dev/null || echo "")
+            if [[ -n "$risk_result" ]]; then
+                local overall_risk
+                overall_risk=$(echo "$risk_result" | jq -r '.overall_risk // 50' 2>/dev/null || echo "50")
+                if [[ "$overall_risk" -gt 80 ]]; then
+                    daemon_log WARN "HIGH RISK (${overall_risk}%) predicted for issue #${issue_num} — upgrading model"
+                    export CLAUDE_MODEL="opus"
+                elif [[ "$overall_risk" -lt 30 ]]; then
+                    daemon_log INFO "LOW RISK (${overall_risk}%) predicted for issue #${issue_num}"
+                fi
+            fi
+        fi
+    fi
+
+    # Check disk space before spawning
+    local free_space_kb
+    free_space_kb=$(df -k "." 2>/dev/null | tail -1 | awk '{print $4}')
+    if [[ -n "$free_space_kb" ]] && [[ "$free_space_kb" -lt 1048576 ]] 2>/dev/null; then
+        daemon_log WARN "Low disk space ($(( free_space_kb / 1024 ))MB) — skipping issue #${issue_num}"
+        return 1
+    fi
+
+    local work_dir="" branch_name="daemon/issue-${issue_num}"
+
+    if [[ "$WATCH_MODE" == "org" && -n "$repo_full_name" ]]; then
+        # Org mode: use cloned repo directory
+        local owner="${repo_full_name%%/*}"
+        local repo="${repo_full_name##*/}"
+        work_dir=$(daemon_ensure_repo "$owner" "$repo") || return 1
+
+        # Create branch in the cloned repo
+        (
+            cd "$work_dir"
+            git checkout -B "$branch_name" "${BASE_BRANCH}" 2>/dev/null
+        ) || {
+            daemon_log ERROR "Failed to create branch in ${repo_full_name}"
+            return 1
+        }
+        daemon_log INFO "Org mode: working in ${work_dir} (${repo_full_name})"
+    else
+        # Standard mode: use git worktree
+        work_dir="${WORKTREE_DIR}/daemon-issue-${issue_num}"
+
+        # Serialize worktree operations with a lock file (run in subshell to auto-close FD)
+        mkdir -p "$WORKTREE_DIR"
+        local wt_ok=0
+        (
+            flock -w 30 200 2>/dev/null || true
+
+            # Clean up stale worktree if it exists
+            if [[ -d "$work_dir" ]]; then
+                git worktree remove "$work_dir" --force 2>/dev/null || true
+            fi
+            git branch -D "$branch_name" 2>/dev/null || true
+
+            git worktree add "$work_dir" -b "$branch_name" "$BASE_BRANCH" 2>/dev/null
+        ) 200>"${WORKTREE_DIR}/.worktree.lock"
+        wt_ok=$?
+
+        if [[ $wt_ok -ne 0 ]]; then
+            daemon_log ERROR "Failed to create worktree for issue #${issue_num}"
+            return 1
+        fi
+        daemon_log INFO "Worktree created at ${work_dir}"
+    fi
+
+    # If template is "composed", copy the composed spec into the worktree
+    if [[ "$PIPELINE_TEMPLATE" == "composed" ]]; then
+        local _src_composed="${REPO_DIR:-.}/.claude/pipeline-artifacts/composed-pipeline.json"
+        if [[ -f "$_src_composed" ]]; then
+            local _dst_artifacts="${work_dir}/.claude/pipeline-artifacts"
+            mkdir -p "$_dst_artifacts"
+            cp "$_src_composed" "$_dst_artifacts/composed-pipeline.json" 2>/dev/null || true
+            daemon_log INFO "Copied composed pipeline spec to worktree"
+        fi
+    fi
+
+    # Build pipeline args
+    local pipeline_args=("start" "--issue" "$issue_num" "--pipeline" "$PIPELINE_TEMPLATE")
+    if [[ "$SKIP_GATES" == "true" ]]; then
+        pipeline_args+=("--skip-gates")
+    fi
+    if [[ -n "$MODEL" ]]; then
+        pipeline_args+=("--model" "$MODEL")
+    fi
+    if [[ "$NO_GITHUB" == "true" ]]; then
+        pipeline_args+=("--no-github")
+    fi
+    # Pass session restart config
+    if [[ "${MAX_RESTARTS_CFG:-0}" -gt 0 ]]; then
+        pipeline_args+=("--max-restarts" "$MAX_RESTARTS_CFG")
+    fi
+    # Pass fast test command
+    if [[ -n "${FAST_TEST_CMD_CFG:-}" ]]; then
+        pipeline_args+=("--fast-test-cmd" "$FAST_TEST_CMD_CFG")
+    fi
+
+    # Append any extra pipeline args (from retry escalation, etc.)
+    if [[ ${#extra_pipeline_args[@]} -gt 0 ]]; then
+        pipeline_args+=("${extra_pipeline_args[@]}")
+    fi
+
+    # Run pipeline in work directory (background)
+    # Ignore SIGHUP so tmux attach/detach and process group changes don't kill the pipeline
+    echo -e "\n\n===== Pipeline run $(date -u +%Y-%m-%dT%H:%M:%SZ) =====" >> "$LOG_DIR/issue-${issue_num}.log" 2>/dev/null || true
+    (
+        trap '' HUP
+        cd "$work_dir"
+        exec "$SCRIPT_DIR/sw-pipeline.sh" "${pipeline_args[@]}"
+    ) >> "$LOG_DIR/issue-${issue_num}.log" 2>&1 200>&- &
+    local pid=$!
+
+    daemon_log INFO "Pipeline started for issue #${issue_num} (PID: ${pid})"
+
+    # Track the job (include repo and goal for org mode)
+    daemon_track_job "$issue_num" "$pid" "$work_dir" "$issue_title" "$repo_full_name" "$issue_goal"
+    emit_event "daemon.spawn" "issue=$issue_num" "pid=$pid" "repo=${repo_full_name:-local}"
+    "$SCRIPT_DIR/sw-tracker.sh" notify "spawn" "$issue_num" 2>/dev/null || true
+
+    # Comment on the issue
+    if [[ "$NO_GITHUB" != "true" ]]; then
+        local gh_args=()
+        if [[ -n "$repo_full_name" ]]; then
+            gh_args+=("--repo" "$repo_full_name")
+        fi
+        gh issue comment "$issue_num" ${gh_args[@]+"${gh_args[@]}"} --body "## 🤖 Pipeline Started
+
+**Delivering:** ${issue_title}
+
+| Field | Value |
+|-------|-------|
+| Template | \`${PIPELINE_TEMPLATE}\` |
+| Branch | \`${branch_name}\` |
+| Repo | \`${repo_full_name:-local}\` |
+| Started | $(now_iso) |
+
+_Progress updates will appear below as the pipeline advances through each stage._" 2>/dev/null || true
+    fi
+}
+
+# ─── Track Job ───────────────────────────────────────────────────────────────
+
+daemon_track_job() {
+    local issue_num="$1" pid="$2" worktree="$3" title="${4:-}" repo="${5:-}" goal="${6:-}"
+
+    # Write to SQLite (non-blocking, best-effort)
+    if type db_save_job &>/dev/null; then
+        local job_id="daemon-${issue_num}-$(now_epoch)"
+        db_save_job "$job_id" "$issue_num" "$title" "$pid" "$worktree" "" "${PIPELINE_TEMPLATE:-autonomous}" "$goal" 2>/dev/null || true
+    fi
+
+    # Always write to JSON state file (primary for now)
+    locked_state_update \
+        --argjson num "$issue_num" \
+        --argjson pid "$pid" \
+        --arg wt "$worktree" \
+        --arg title "$title" \
+        --arg started "$(now_iso)" \
+        --arg repo "$repo" \
+        --arg goal "$goal" \
+        '.active_jobs += [{
+            issue: $num,
+            pid: $pid,
+            worktree: $wt,
+            title: $title,
+            started_at: $started,
+            repo: $repo,
+            goal: $goal
+        }]'
+}
+
+# ─── Reap Completed Jobs ────────────────────────────────────────────────────
+
+daemon_reap_completed() {
+    if [[ ! -f "$STATE_FILE" ]]; then
+        return
+    fi
+
+    local jobs
+    jobs=$(jq -c '.active_jobs[]' "$STATE_FILE" 2>/dev/null || true)
+    if [[ -z "$jobs" ]]; then
+        return
+    fi
+
+    local _retry_spawned_for=""
+
+    while IFS= read -r job; do
+        local issue_num pid worktree
+        issue_num=$(echo "$job" | jq -r '.issue // empty')
+        pid=$(echo "$job" | jq -r '.pid // empty')
+        worktree=$(echo "$job" | jq -r '.worktree // empty')
+
+        # Skip malformed entries (corrupted state file)
+        [[ -z "$issue_num" || ! "$issue_num" =~ ^[0-9]+$ ]] && continue
+        [[ -z "$pid" || ! "$pid" =~ ^[0-9]+$ ]] && continue
+
+        # Check if process is still running
+        if kill -0 "$pid" 2>/dev/null; then
+            continue
+        fi
+
+        # Process is dead — determine exit code
+        # Note: wait returns 127 if process was already reaped (e.g., by init)
+        # In that case, check pipeline log for success/failure indicators
+        local exit_code=0
+        wait "$pid" 2>/dev/null || exit_code=$?
+        if [[ "$exit_code" -eq 127 ]]; then
+            # Process already reaped — check log file for real outcome
+            local issue_log="$LOG_DIR/issue-${issue_num}.log"
+            if [[ -f "$issue_log" ]]; then
+                if grep -q "Pipeline completed successfully" "$issue_log" 2>/dev/null; then
+                    exit_code=0
+                elif grep -q "Pipeline failed\|ERROR.*stage.*failed\|exited with status" "$issue_log" 2>/dev/null; then
+                    exit_code=1
+                else
+                    daemon_log WARN "Could not determine exit code for issue #${issue_num} (PID ${pid} already reaped) — marking as failure"
+                    exit_code=1
+                fi
+            else
+                exit_code=1
+            fi
+        fi
+
+        local started_at duration_str="" start_epoch=0 end_epoch=0
+        started_at=$(echo "$job" | jq -r '.started_at // empty')
+        if [[ -n "$started_at" ]]; then
+            # macOS date -j for parsing ISO dates (TZ=UTC to parse Z-suffix correctly)
+            start_epoch=$(TZ=UTC date -j -f "%Y-%m-%dT%H:%M:%SZ" "$started_at" +%s 2>/dev/null || date -d "$started_at" +%s 2>/dev/null || echo "0")
+            end_epoch=$(now_epoch)
+            if [[ "$start_epoch" -gt 0 ]]; then
+                duration_str=$(format_duration $((end_epoch - start_epoch)))
+            fi
+        fi
+
+        local result_str="success"
+        [[ "$exit_code" -ne 0 ]] && result_str="failure"
+        local dur_s=0
+        [[ "$start_epoch" -gt 0 ]] && dur_s=$((end_epoch - start_epoch))
+        emit_event "daemon.reap" "issue=$issue_num" "result=$result_str" "duration_s=$dur_s"
+
+        # Update SQLite (mark job complete/failed)
+        if type db_complete_job &>/dev/null && type db_fail_job &>/dev/null; then
+            local _db_job_id="daemon-${issue_num}-${start_epoch}"
+            if [[ "$exit_code" -eq 0 ]]; then
+                db_complete_job "$_db_job_id" "$result_str" 2>/dev/null || true
+            else
+                db_fail_job "$_db_job_id" "$result_str" 2>/dev/null || true
+            fi
+        fi
+
+        if [[ "$exit_code" -eq 0 ]]; then
+            daemon_on_success "$issue_num" "$duration_str"
+        else
+            daemon_on_failure "$issue_num" "$exit_code" "$duration_str"
+
+            # Cancel any lingering in_progress GitHub Check Runs for failed job
+            if [[ "${NO_GITHUB:-false}" != "true" && -n "$worktree" ]]; then
+                local check_ids_file="${worktree}/.claude/pipeline-artifacts/check-run-ids.json"
+                if [[ -f "$check_ids_file" ]]; then
+                    daemon_log INFO "Cancelling in-progress check runs for issue #${issue_num}"
+                    local _stage
+                    while IFS= read -r _stage; do
+                        [[ -z "$_stage" ]] && continue
+                        # Direct API call since we're in daemon context
+                        local _run_id
+                        _run_id=$(jq -r --arg s "$_stage" '.[$s] // empty' "$check_ids_file" 2>/dev/null || true)
+                        if [[ -n "$_run_id" && "$_run_id" != "null" ]]; then
+                            local _detected
+                            _detected=$(git remote get-url origin 2>/dev/null | sed 's|.*github.com[:/]\(.*\)\.git$|\1|' || true)
+                            if [[ -n "$_detected" ]]; then
+                                local _owner="${_detected%%/*}" _repo="${_detected##*/}"
+                                gh api "repos/${_owner}/${_repo}/check-runs/${_run_id}" \
+                                    --method PATCH \
+                                    --field status=completed \
+                                    --field conclusion=cancelled \
+                                    --silent 2>/dev/null || true
+                            fi
+                        fi
+                    done < <(jq -r 'keys[]' "$check_ids_file" 2>/dev/null || true)
+                fi
+            fi
+        fi
+
+        # Finalize memory (capture failure patterns for future runs)
+        if type memory_finalize_pipeline &>/dev/null 2>&1; then
+            local _job_state _job_artifacts
+            _job_state="${worktree:-.}/.claude/pipeline-state.md"
+            _job_artifacts="${worktree:-.}/.claude/pipeline-artifacts"
+            memory_finalize_pipeline "$_job_state" "$_job_artifacts" 2>/dev/null || true
+        fi
+
+        # Clean up progress tracking for this job
+        daemon_clear_progress "$issue_num"
+
+        # Release claim lock (label-based coordination)
+        local reap_machine_name
+        reap_machine_name=$(jq -r '.machines[] | select(.role == "primary") | .name' "$HOME/.shipwright/machines.json" 2>/dev/null || hostname -s)
+        release_claim "$issue_num" "$reap_machine_name"
+
+        # Always remove the OLD job entry from active_jobs to prevent
+        # re-reaping of the dead PID on the next cycle.  When a retry was
+        # spawned, daemon_spawn_pipeline already added a fresh entry with
+        # the new PID — we must not leave the stale one behind.
+        locked_state_update --argjson num "$issue_num" \
+            --argjson old_pid "${pid:-0}" \
+            '.active_jobs = [.active_jobs[] | select(.issue != $num or .pid != $old_pid)]'
+        untrack_priority_job "$issue_num"
+
+        if [[ "$_retry_spawned_for" == "$issue_num" ]]; then
+            daemon_log INFO "Retry spawned for issue #${issue_num} — skipping worktree cleanup"
+        else
+            # Clean up worktree (skip for org-mode clones — they persist)
+            local job_repo
+            job_repo=$(echo "$job" | jq -r '.repo // ""')
+            if [[ -z "$job_repo" ]] && [[ -d "$worktree" ]]; then
+                git worktree remove "$worktree" --force 2>/dev/null || true
+                daemon_log INFO "Cleaned worktree: $worktree"
+                git branch -D "daemon/issue-${issue_num}" 2>/dev/null || true
+            elif [[ -n "$job_repo" ]]; then
+                daemon_log INFO "Org-mode: preserving clone for ${job_repo}"
+            fi
+        fi
+
+        # Dequeue next issue if available AND we have capacity
+        # NOTE: locked_get_active_count prevents TOCTOU race with the
+        # active_jobs removal above.  A tiny window remains between
+        # the count read and dequeue_next's own lock acquisition, but
+        # dequeue_next is itself locked, so the worst case is a
+        # missed dequeue that the next poll cycle will pick up.
+        local current_active
+        current_active=$(locked_get_active_count)
+        if [[ "$current_active" -lt "$MAX_PARALLEL" ]]; then
+            local next_issue
+            next_issue=$(dequeue_next)
+            if [[ -n "$next_issue" ]]; then
+                local next_title
+                next_title=$(jq -r --arg n "$next_issue" '.titles[$n] // ""' "$STATE_FILE" 2>/dev/null || true)
+                daemon_log INFO "Dequeuing issue #${next_issue}: ${next_title}"
+                daemon_spawn_pipeline "$next_issue" "$next_title"
+            fi
+        fi
+    done <<< "$jobs"
+}
+
+# ─── Success Handler ────────────────────────────────────────────────────────
+
+daemon_on_success() {
+    local issue_num="$1" duration="${2:-}"
+
+    # Reset consecutive failure tracking on any success
+    reset_failure_tracking
+
+    daemon_log SUCCESS "Pipeline completed for issue #${issue_num} (${duration:-unknown})"
+
+    # Record pipeline duration for adaptive threshold learning
+    if [[ -n "$duration" && "$duration" != "unknown" ]]; then
+        # Parse duration string back to seconds (e.g. "5m 30s" → 330)
+        local dur_secs=0
+        local _h _m _s
+        _h=$(echo "$duration" | grep -oE '[0-9]+h' | grep -oE '[0-9]+' || true)
+        _m=$(echo "$duration" | grep -oE '[0-9]+m' | grep -oE '[0-9]+' || true)
+        _s=$(echo "$duration" | grep -oE '[0-9]+s' | grep -oE '[0-9]+' || true)
+        dur_secs=$(( ${_h:-0} * 3600 + ${_m:-0} * 60 + ${_s:-0} ))
+        if [[ "$dur_secs" -gt 0 ]]; then
+            record_pipeline_duration "$PIPELINE_TEMPLATE" "$dur_secs" "success"
+            record_scaling_outcome "$MAX_PARALLEL" "success"
+        fi
+    fi
+
+    # Record in completed list + clear retry count for this issue
+    locked_state_update \
+        --argjson num "$issue_num" \
+        --arg result "success" \
+        --arg dur "${duration:-unknown}" \
+        --arg completed_at "$(now_iso)" \
+        '.completed += [{
+            issue: $num,
+            result: $result,
+            duration: $dur,
+            completed_at: $completed_at
+        }] | .completed = .completed[-500:]
+        | del(.retry_counts[($num | tostring)])'
+
+    if [[ "$NO_GITHUB" != "true" ]]; then
+        # Remove watch label, add success label
+        gh issue edit "$issue_num" \
+            --remove-label "$ON_SUCCESS_REMOVE_LABEL" \
+            --add-label "$ON_SUCCESS_ADD_LABEL" 2>/dev/null || true
+
+        # Comment on issue
+        gh issue comment "$issue_num" --body "## ✅ Pipeline Complete
+
+The autonomous pipeline finished successfully.
+
+| Field | Value |
+|-------|-------|
+| Duration | ${duration:-unknown} |
+| Completed | $(now_iso) |
+
+Check the associated PR for the implementation." 2>/dev/null || true
+
+        # Optionally close the issue
+        if [[ "$ON_SUCCESS_CLOSE_ISSUE" == "true" ]]; then
+            gh issue close "$issue_num" 2>/dev/null || true
+        fi
+    fi
+
+    notify "Pipeline Complete — Issue #${issue_num}" \
+        "Duration: ${duration:-unknown}" "success"
+    "$SCRIPT_DIR/sw-tracker.sh" notify "completed" "$issue_num" 2>/dev/null || true
+
+    # PM agent: record success for learning
+    if [[ -x "$SCRIPT_DIR/sw-pm.sh" ]]; then
+        bash "$SCRIPT_DIR/sw-pm.sh" learn "$issue_num" success 2>/dev/null || true
+    fi
+}
+
+# ─── Failure Classification ─────────────────────────────────────────────────
+