shipwright-cli 1.9.0 → 1.10.0

package/scripts/sw-memory.sh

@@ -6,7 +6,7 @@
 set -euo pipefail
 trap 'echo "ERROR: $BASH_SOURCE:$LINENO exited with status $?" >&2' ERR
 
-VERSION="1.9.0"
+VERSION="1.10.0"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
 
@@ -218,7 +218,7 @@ memory_capture_failure() {
 
     (
         if command -v flock &>/dev/null; then
-            flock -w 10 200 2>/dev/null || { warn "Memory lock timeout"; }
+            flock -w 10 200 2>/dev/null || { warn "Memory lock timeout"; return 1; }
         fi
         local tmp_file
         tmp_file=$(mktemp "${failures_file}.tmp.XXXXXX")
@@ -284,7 +284,7 @@ memory_record_fix_outcome() {
 
     (
         if command -v flock &>/dev/null; then
-            flock -w 10 200 2>/dev/null || { warn "Memory lock timeout"; }
+            flock -w 10 200 2>/dev/null || { warn "Memory lock timeout"; return 1; }
         fi
         local tmp_file
         tmp_file=$(mktemp "${failures_file}.tmp.XXXXXX")
@@ -312,6 +312,178 @@ memory_record_fix_outcome() {
         "resolved=${fix_resolved}"
 }
 
+# memory_track_fix <error_sig> <success_bool>
+# Convenience wrapper for memory_record_fix_outcome
+memory_track_fix() {
+    local error_sig="${1:-}"
+    local success="${2:-false}"
+    [[ -z "$error_sig" ]] && return 0
+    memory_record_fix_outcome "$error_sig" "true" "$success" 2>/dev/null || true
+}
+
+# memory_query_fix_for_error <error_pattern>
+# Searches failure memory for known fixes matching the given error pattern.
+# Returns JSON with the best fix (highest effectiveness rate) or empty.
+memory_query_fix_for_error() {
+    local error_pattern="$1"
+    [[ -z "$error_pattern" ]] && return 0
+
+    ensure_memory_dir
+    local mem_dir
+    mem_dir="$(repo_memory_dir)"
+    local failures_file="$mem_dir/failures.json"
+
+    [[ ! -f "$failures_file" ]] && return 0
+
+    # Search for matching failures with successful fixes
+    local matches
+    matches=$(jq -r --arg pat "$error_pattern" '
+        [.failures[]
+        | select(.pattern != null and .pattern != "")
+        | select(.pattern | test($pat; "i") // false)
+        | select(.fix != null and .fix != "")
+        | select((.fix_effectiveness_rate // 0) > 30)
+        | {fix, fix_effectiveness_rate, seen_count, category, stage, pattern}]
+        | sort_by(-.fix_effectiveness_rate)
+        | .[0] // null
+    ' "$failures_file" 2>/dev/null) || true
+
+    if [[ -n "$matches" && "$matches" != "null" ]]; then
+        echo "$matches"
+    fi
+}
+
+# memory_closed_loop_inject <error_sig>
+# Combines error → memory → fix into injectable text for build retries.
+# Returns a one-line summary suitable for goal augmentation.
+memory_closed_loop_inject() {
+    local error_sig="$1"
+    [[ -z "$error_sig" ]] && return 0
+
+    local fix_json
+    fix_json=$(memory_query_fix_for_error "$error_sig") || true
+    [[ -z "$fix_json" || "$fix_json" == "null" ]] && return 0
+
+    local fix_text success_rate category
+    fix_text=$(echo "$fix_json" | jq -r '.fix // ""')
+    success_rate=$(echo "$fix_json" | jq -r '.fix_effectiveness_rate // 0')
+    category=$(echo "$fix_json" | jq -r '.category // "unknown"')
+
+    [[ -z "$fix_text" ]] && return 0
+
+    echo "[$category, ${success_rate}% success rate] $fix_text"
+}
+
+memory_capture_failure_from_log() {
+    local artifacts_dir="${1:-}"
+    local error_log="${artifacts_dir}/error-log.jsonl"
+    [[ ! -f "$error_log" ]] && return 0
+
+    ensure_memory_dir
+    local mem_dir
+    mem_dir="$(repo_memory_dir)"
+    local failures_file="$mem_dir/failures.json"
+
+    # Read last 50 entries
+    local entries
+    entries=$(tail -50 "$error_log" 2>/dev/null) || return 0
+    [[ -z "$entries" ]] && return 0
+
+    local captured=0
+    while IFS= read -r line; do
+        [[ -z "$line" ]] && continue
+
+        local err_type err_text
+        err_type=$(echo "$line" | jq -r '.type // "unknown"' 2>/dev/null) || continue
+        err_text=$(echo "$line" | jq -r '.error // ""' 2>/dev/null) || continue
+        [[ -z "$err_text" ]] && continue
+
+        # Deduplicate: skip if this exact pattern already exists in failures
+        local pattern_short
+        pattern_short=$(echo "$err_text" | head -1 | cut -c1-200)
+        local already_exists
+        already_exists=$(jq --arg pat "$pattern_short" \
+            '[.failures[] | select(.pattern == $pat)] | length' \
+            "$failures_file" 2>/dev/null || echo "0")
+        if [[ "${already_exists:-0}" -gt 0 ]]; then
+            continue
+        fi
+
+        # Feed into memory_capture_failure with the error type as stage
+        memory_capture_failure "$err_type" "$err_text" 2>/dev/null || true
+        captured=$((captured + 1))
+    done <<< "$entries"
+
+    if [[ "$captured" -gt 0 ]]; then
+        emit_event "memory.error_log_processed" "captured=$captured"
+    fi
+}
+
+# _memory_aggregate_global
+# Promotes high-frequency failure patterns to global.json for cross-repo learning
+_memory_aggregate_global() {
+    ensure_memory_dir
+    local mem_dir
+    mem_dir="$(repo_memory_dir)"
+    local failures_file="$mem_dir/failures.json"
+    [[ ! -f "$failures_file" ]] && return 0
+
+    local global_file="$GLOBAL_MEMORY"
+    [[ ! -f "$global_file" ]] && return 0
+
+    # Find patterns with seen_count >= 3
+    local frequent_patterns
+    frequent_patterns=$(jq -r '.failures[] | select(.seen_count >= 3) | .pattern' \
+        "$failures_file" 2>/dev/null) || return 0
+    [[ -z "$frequent_patterns" ]] && return 0
+
+    local promoted=0
+    while IFS= read -r pattern; do
+        [[ -z "$pattern" ]] && continue
+
+        # Check if already in global
+        local exists
+        exists=$(jq --arg p "$pattern" \
+            '[.common_patterns[] | select(.pattern == $p)] | length' \
+            "$global_file" 2>/dev/null || echo "0")
+        if [[ "${exists:-0}" -gt 0 ]]; then
+            continue
+        fi
+
+        # Add to global, cap at 100 entries
+        local tmp_global
+        tmp_global=$(mktemp "${global_file}.tmp.XXXXXX")
+        jq --arg p "$pattern" \
+           --arg ts "$(now_iso)" \
+           --arg cat "general" \
+           '.common_patterns += [{pattern: $p, promoted_at: $ts, category: $cat, source: "aggregate"}] |
+            .common_patterns = (.common_patterns | .[-100:])' \
+           "$global_file" > "$tmp_global" && mv "$tmp_global" "$global_file" || rm -f "$tmp_global"
+        promoted=$((promoted + 1))
+    done <<< "$frequent_patterns"
+
+    if [[ "$promoted" -gt 0 ]]; then
+        emit_event "memory.global_aggregated" "promoted=$promoted"
+    fi
+}
+
+# memory_finalize_pipeline <state_file> <artifacts_dir>
+# Single call that closes multiple feedback loops at pipeline completion
+memory_finalize_pipeline() {
+    local state_file="${1:-}"
+    local artifacts_dir="${2:-}"
+    [[ -z "$state_file" || ! -f "$state_file" ]] && return 0
+
+    # Step 1: Capture pipeline-level learnings
+    memory_capture_pipeline "$state_file" "$artifacts_dir" 2>/dev/null || true
+
+    # Step 2: Process error log into failures.json
+    memory_capture_failure_from_log "$artifacts_dir" 2>/dev/null || true
+
+    # Step 3: Aggregate high-frequency patterns to global memory
+    _memory_aggregate_global 2>/dev/null || true
+}
+
 # memory_analyze_failure <log_file> <stage>
 # Uses Claude to analyze a pipeline failure and fill in root_cause/fix/category.
 memory_analyze_failure() {
@@ -793,6 +965,29 @@ memory_inject_context() {
             ;;
     esac
 
+    # ── Cross-repo memory injection (global learnings) ──
+    if [[ -f "$GLOBAL_MEMORY" ]]; then
+        local global_patterns
+        global_patterns=$(jq -r --arg stage "$stage_id" '
+            .common_patterns // [] | .[] |
+            select(.category == $stage or .category == "general" or .category == null) |
+            .summary // .description // empty
+        ' "$GLOBAL_MEMORY" 2>/dev/null | head -5 || true)
+
+        local cross_repo_learnings
+        cross_repo_learnings=$(jq -r '
+            .cross_repo_learnings // [] | .[-5:][] |
+            "- [\(.repo // "unknown")] \(.type // "learning"): bugs=\(.bugs // 0), warnings=\(.warnings // 0)"
+        ' "$GLOBAL_MEMORY" 2>/dev/null | head -5 || true)
+
+        if [[ -n "$global_patterns" || -n "$cross_repo_learnings" ]]; then
+            echo ""
+            echo "## Cross-Repo Learnings (Global)"
+            [[ -n "$global_patterns" ]] && echo "$global_patterns"
+            [[ -n "$cross_repo_learnings" ]] && echo "$cross_repo_learnings"
+        fi
+    fi
+
     echo ""
     emit_event "memory.inject" "stage=${stage_id}"
 }

package/scripts/sw-pipeline-composer.sh

@@ -6,7 +6,7 @@
 set -euo pipefail
 trap 'echo "ERROR: $BASH_SOURCE:$LINENO exited with status $?" >&2' ERR
 
-VERSION="1.9.0"
+VERSION="1.10.0"
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 REPO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
 
@@ -114,7 +114,7 @@ composer_create_pipeline() {
         else
             repo_context="$ci_history"
         fi
-        info "CI history: p90 duration=${p90_timeout}s — using for timeout tuning"
+        info "CI history: p90 duration=${p90_timeout}s — using for timeout tuning" >&2
     fi
 
     # Try intelligence-driven composition
@@ -122,7 +122,7 @@ composer_create_pipeline() {
        [[ -n "$issue_analysis" ]] && \
        type intelligence_compose_pipeline &>/dev/null; then
 
-        info "Composing pipeline with intelligence engine..."
+        info "Composing pipeline with intelligence engine..." >&2
 
         local composed=""
         composed=$(intelligence_compose_pipeline "$issue_analysis" "$repo_context" "$budget_json" 2>/dev/null) || true
@@ -138,7 +138,7 @@ composer_create_pipeline() {
 
                 local stage_count
                 stage_count=$(echo "$composed" | jq '.stages | length')
-                success "Composed pipeline: ${stage_count} stages"
+                success "Composed pipeline: ${stage_count} stages" >&2
 
                 emit_event "composer.created" \
                     "stages=${stage_count}" \
@@ -148,17 +148,17 @@ composer_create_pipeline() {
                 echo "$output_file"
                 return 0
             else
-                warn "Intelligence pipeline failed validation, falling back to template"
+                warn "Intelligence pipeline failed validation, falling back to template" >&2
             fi
         else
-            warn "Intelligence composition returned invalid JSON, falling back to template"
+            warn "Intelligence composition returned invalid JSON, falling back to template" >&2
         fi
     fi
 
     # Fallback: use static template
     local fallback_template="${TEMPLATES_DIR}/standard.json"
     if [[ -f "$fallback_template" ]]; then
-        info "Using fallback template: standard"
+        info "Using fallback template: standard" >&2
         local tmp_file
         tmp_file=$(mktemp "${output_file}.XXXXXX")
         cp "$fallback_template" "$tmp_file"
@@ -173,7 +173,7 @@ composer_create_pipeline() {
         return 0
     fi
 
-    error "No templates available for fallback"
+    error "No templates available for fallback" >&2
     return 1
 }