ruvnet-brain 4.7.18 → 4.7.20

package/scripts/job-heartbeat.sh

@@ -81,7 +81,7 @@ PREV_HB=""
 
 # Start receipt. If the job vanishes without ever writing an end receipt, THIS is the evidence that
 # it started and never finished — a state the watchdog reports as FAILING, not as silence.
-write_receipt "{\"label\":\"$LABEL\",\"started_at\":\"$STARTED\",\"state\":\"running\",\"pid\":$$,\"run_id\":\"$RUN_ID\",\"command\":\"$(echo "$@" | sed 's/"/\\"/g')\"}"
+write_receipt "{\"label\":\"$LABEL\",\"started_at\":\"$STARTED\",\"state\":\"running\",\"pid\":$$,\"run_id\":\"$RUN_ID\",\"command\":\"$(printf '%s %s' "$1" "${2:-}" | tr -d '"\\\n\r' | cut -c1-200)\"}"
 
 notify() { # notify <title> <body> <priority>
   # NTFY_TOPIC set-but-EMPTY is an explicit opt-out (2026-07-18): unit tests wrap this script around
@@ -92,8 +92,10 @@ notify() { # notify <title> <body> <priority>
   if [ "${NTFY_TOPIC+set}" = "set" ] && [ -z "$NTFY_TOPIC" ]; then return 0; fi
   topic="${NTFY_TOPIC:-}"
   [ -z "$topic" ] && [ -f "$HOME/.cache/ruvnet-brain/ntfy-topic" ] && topic="$(cat "$HOME/.cache/ruvnet-brain/ntfy-topic")"
-  [ -z "$topic" ] && return 0
-  curl -sS -m 10 -H "Title: $1" -H "Priority: $3" -H "Tags: rotating_light" -d "$2" "https://ntfy.sh/$topic" >/dev/null 2>&1 || true
+  [ -z "$topic" ] && return 1
+  # --fail: an HTTP error is a failed delivery. The caller records whether the page landed, so a page that
+  # never arrived is retried on the next run instead of muting the whole failing streak (review 2026-10-25).
+  curl -sS --fail -m 10 -H "Title: $1" -H "Priority: $3" -H "Tags: ${4:-rotating_light}" -d "$2" "https://ntfy.sh/$topic" >/dev/null 2>&1
 }
 
 # The run_id currently on disk for this label, or empty if the receipt is missing/unreadable/has
@@ -104,6 +106,17 @@ current_run_id() {
 
 finish() {
   code=${FORCED_CODE:-$?}
+  # Stop the run-time timer FIRST, on every exit path (skip, stale writer, normal), so it can never outlive
+  # this wrapper (Opus review 2026-10-25).
+  [ -n "${TIMER:-}" ] && kill "$TIMER" 2>/dev/null
+  timed_out=0
+  if [ -f "$HB.timeout-$RUN_ID" ]; then
+    # Anything the overrunning job left behind (started during its TERM handling, say) goes now.
+    timed_out=1; stop_job; rm -f "$HB.timeout-$RUN_ID"
+  fi
+  # An overrun is a failure however the job chose to exit on TERM, including 0 and 75 (the skip code), so this
+  # comes before the skip branch (GPT-6.1 review round 2).
+  [ "$timed_out" -eq 1 ] && [ "$code" -le 128 ] && code=143
   ended="$(ts)"
   dur=$(( $(date +%s) - START_EPOCH ))
   # Exit 75 = SKIP (lock held by a live run). Restore the pre-fire receipt so the live run's evidence
@@ -132,15 +145,36 @@ finish() {
   elif [ "$code" -gt 128 ]; then
     sig=$((code - 128))
     state="killed"; extra=",\"signal\":$sig"
+    [ "$timed_out" -eq 1 ] && extra="$extra,\"timed_out_after_sec\":$MAX_SECONDS"
   else
     state="failed"; extra=""
   fi
-  write_receipt "{\"label\":\"$LABEL\",\"started_at\":\"$STARTED\",\"ended_at\":\"$ended\",\"state\":\"$state\",\"exit_code\":$code,\"duration_sec\":$dur,\"run_id\":\"$RUN_ID\"$extra}"
-  # Gong on failure, immediately — not at the next watchdog sweep. A failing nightly should reach the
-  # phone while it is still tonight's problem.
+  # PAGE ON THE TRANSITION, NOT ON EVERY RUN (owner, 2026-10-25: 10,747 notifications on one topic). A job that
+  # starts failing pages once, urgently, while it is still tonight's problem; while it KEEPS failing it stays
+  # quiet (failing_since carries the first failure; the nightly watchdog's digest still lists it); the run
+  # that passes again sends one low-priority "recovered" note. The previous run's state comes from PREV_HB.
+  prev_state="$(printf '%s' "$PREV_HB" | sed -n 's/.*"state":"\([^"]*\)".*/\1/p' | head -1)"
+  prev_since="$(printf '%s' "$PREV_HB" | sed -n 's/.*"failing_since":"\([^"]*\)".*/\1/p' | head -1)"
+  was_failing=0; case "$prev_state" in failed|killed) was_failing=1 ;; esac
+  # A failing streak counts as already paged only if that page was delivered ("paged":false = retry it).
+  # Only a page PROVEN delivered ("paged":true) keeps the streak quiet; a receipt from before this field existed
+  # counts as unknown and is paged once more (review round 2).
+  prev_unpaged=1; case "$PREV_HB" in *'"paged":true'*) prev_unpaged=0 ;; esac
+  since=""
   if [ "$code" -ne 0 ]; then
-    notify "🔴 SCHEDULED JOB FAILED: $LABEL" "exit $code after ${dur}s — see the job's log. Receipt: $HB" "urgent"
+    if [ "$was_failing" -eq 1 ] && [ -n "$prev_since" ]; then since="$prev_since"; else since="$STARTED"; fi
+    extra="$extra,\"failing_since\":\"$since\""
   fi
+  if [ "$code" -ne 0 ] && { [ "$was_failing" -eq 0 ] || [ "$prev_unpaged" -eq 1 ]; }; then
+    reason="exit $code after ${dur}s"; [ "$timed_out" -eq 1 ] && reason="stopped after running past its ${MAX_SECONDS}s limit"
+    if notify "🔴 SCHEDULED JOB FAILED: $LABEL" "$reason — see the job's log. Receipt: $HB. Further failures stay quiet until it recovers." "urgent"; then
+      extra="$extra,\"paged\":true"; else extra="$extra,\"paged\":false"; fi
+  elif [ "$code" -ne 0 ]; then
+    extra="$extra,\"paged\":true"
+  elif [ "$was_failing" -eq 1 ]; then
+    notify "✅ Recovered: $LABEL" "passed again after failing since ${prev_since:-an earlier run}" "low" "white_check_mark" || true
+  fi
+  write_receipt "{\"label\":\"$LABEL\",\"started_at\":\"$STARTED\",\"ended_at\":\"$ended\",\"state\":\"$state\",\"exit_code\":$code,\"duration_sec\":$dur,\"run_id\":\"$RUN_ID\"$extra}"
   exit "$code"
 }
 trap finish EXIT
@@ -148,8 +182,46 @@ trap finish EXIT
 # The exit codes (143 = 128+15 TERM, 130 = 128+2 INT) intentionally reuse the same 128+signal
 # convention `wait` uses for a directly-signalled child, so finish() classifies BOTH as "killed"
 # with the right signal number, whether the signal reached the child directly or via this wrapper.
-trap 'FORCED_CODE=143; kill -TERM "$CHILD" 2>/dev/null; exit 143' TERM
-trap 'FORCED_CODE=130; kill -TERM "$CHILD" 2>/dev/null; exit 130' INT
+# The job stays in this wrapper's process group, so launchd's own cleanup of a dead job (it kills the job's whole
+# group) still covers everything, even if this wrapper is SIGKILLed (Opus review round 3: a separate group escaped it).
+# job_pids: every process the job owns. Under launchd this wrapper leads its group, so that is every group member
+# except this wrapper, its timer, and this wrapper's own helper processes (ps/awk/command substitutions) — which
+# catches processes the job started late or that were reparented. Run any other way (a terminal, a test harness),
+# it falls back to the job's current process tree.
+OWN_GROUP=0; [ "$(ps -o pgid= -p $$ 2>/dev/null | tr -d ' ')" = "$$" ] && OWN_GROUP=1
+job_pids() {
+  if [ "$OWN_GROUP" = 1 ]; then
+    ps -A -o pid= -o ppid= -o pgid= 2>/dev/null | awk -v me=$$ -v t="${TIMER:-0}" -v c="${CHILD:-0}" '
+      { pid[NR]=$1; pp[NR]=$2; pg[NR]=$3 }
+      END { helper[t]=1
+            for (i=1;i<=NR;i++) if (pp[i]==me && pid[i]!=c) helper[pid[i]]=1
+            for (i=1;i<=NR;i++) if (pg[i]==me && pid[i]!=me && !(pid[i] in helper) && !(pp[i] in helper)) print pid[i] }'
+  else
+    tree_of "$CHILD"
+  fi
+}
+tree_of() { kill -0 "$1" 2>/dev/null && printf '%s ' "$1"
+  for k in $(ps -A -o pid= -o ppid= 2>/dev/null | awk -v p="$1" '$2==p {print $1}'); do tree_of "$k"; done; }
+# stop_job: TERM everything the job owns, give it up to 10 s (inside launchd's 20 s exit allowance), then KILL.
+# Every pid seen during the stop is remembered, so a descendant that outlives its parent (and so drops out of the
+# process tree in the fallback mode) still gets the final KILL. The full guarantee, including processes started
+# later, holds when this wrapper leads its group, as under launchd.
+stop_job() {
+  seen="$(job_pids)"; [ -n "$seen" ] && kill -TERM $seen 2>/dev/null
+  i=0; while [ "$i" -lt 10 ]; do
+    now_pids="$(job_pids)"; seen="$seen $now_pids"
+    alive=""; for p in $seen; do kill -0 "$p" 2>/dev/null && alive="$alive $p"; done
+    [ -z "$alive" ] && return 0
+    sleep 1; i=$((i + 1))
+  done
+  for p in $seen $(job_pids); do kill -KILL "$p" 2>/dev/null; done
+  return 0
+}
+trap 'FORCED_CODE=143; [ -n "${TIMER:-}" ] && kill "$TIMER" 2>/dev/null; stop_job; exit 143' TERM
+trap 'FORCED_CODE=130; [ -n "${TIMER:-}" ] && kill "$TIMER" 2>/dev/null; stop_job; exit 130' INT
+# The timer never kills anything itself: it marks the overrun and signals this wrapper, which stops the job.
+OVERRUN=0
+trap 'OVERRUN=1' USR1
 
 # Run the job in the BACKGROUND and `wait` for it — do NOT run it in the foreground.
 # Break-test finding (2026-07-13): a POSIX shell blocked on a FOREGROUND child does not run its trap
@@ -160,4 +232,29 @@ trap 'FORCED_CODE=130; kill -TERM "$CHILD" 2>/dev/null; exit 130' INT
 # ("started and never finished"). Trap for catchable deaths, watchdog for uncatchable ones.
 "$@" &
 CHILD=$!
-wait "$CHILD"
+# RUN-TIME LIMIT (2026-10-25: five jobs finished their work and then never exited, each holding its own
+# next run off for 7-9 hours). After JOB_MAX_SECONDS (default 2 h; a plist may set less) the job is
+# stopped: TERM, then KILL 10 s later. It is recorded as killed and pages once like any other failure.
+MAX_SECONDS="${JOB_MAX_SECONDS:-7200}"
+WRAPPER_PID=$$
+case "$MAX_SECONDS" in ''|*[!0-9]*) MAX_SECONDS=7200 ;; esac
+[ "$MAX_SECONDS" -lt 2 ] && MAX_SECONDS=2
+(
+  # Short sleeps, not one long one: killing this shell then leaves at most a 2 s sleep behind, never a 2 h one
+  # holding the job's output open (review round 2).
+  # A wrapper that is gone (SIGKILLed: launchd then cleans the group itself) ends the timer at the next tick.
+  n=0; while [ "$n" -lt "$MAX_SECONDS" ]; do sleep 2; n=$((n + 2)); kill -0 "$WRAPPER_PID" 2>/dev/null || exit 0; done
+  # Only for OUR child: still alive and still parented by this wrapper. If the PID was reused, do nothing.
+  [ "$(ps -o ppid= -p "$CHILD" 2>/dev/null | tr -d ' ')" = "$WRAPPER_PID" ] || exit 0
+  : > "$HB.timeout-$RUN_ID"
+  kill -USR1 "$WRAPPER_PID" 2>/dev/null
+) </dev/null >/dev/null 2>&1 &
+TIMER=$!
+# `wait` returns early when USR1 arrives; stop the job, then wait again for its real exit status.
+while :; do
+  if [ "$OVERRUN" = 1 ]; then OVERRUN=0; stop_job; fi
+  wait "$CHILD"; RC=$?
+  if [ "$OVERRUN" = 1 ]; then OVERRUN=0; stop_job; continue; fi
+  break
+done
+exit "$RC"

package/scripts/nightly-gists.sh

@@ -17,7 +17,10 @@
 set -eu
 
 cd "$(dirname "$0")/.."
-export PATH="/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin"
+# The owner's macOS-approved node (~/.local/share/ruvnet-brain/job-bin) and Apple's tools come first: this repo and
+# the model cache live on an external volume, and launchd-started Homebrew node is held at its first read there
+# (2026-10-25: every embedding shard "stalled" for 15 min, then the run failed after 1 h 49 min).
+export PATH="$HOME/.local/share/ruvnet-brain/job-bin:/usr/bin:/bin:/opt/homebrew/bin:/usr/local/bin"
 export KB_MODEL_CACHE="${KB_MODEL_CACHE:-/Users/stuartkerr/Code/PowerPlatePulse/scripts/models-cache}"
 
 mkdir -p logs