ruvnet-brain 4.7.19 → 4.7.21

package/scripts/issue-watch.mjs

@@ -33,6 +33,7 @@ import path from 'node:path';
 import os from 'node:os';
 import { spawnSync } from 'node:child_process';
 import { fileURLToPath, pathToFileURL } from 'node:url';
+import { isDirectInvocation } from '../plugin/scripts/entrypoint.mjs';
 
 const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const REPO = 'stuinfla/ruvnet-brain';
@@ -46,6 +47,23 @@ export const OWNER_LOGIN = 'stuinfla';
 // moved here.)
 export const BOT_MARKER = '🤖 Automated';
 const SLA_HOURS = 4;
+// A person's issue whose detail AND list comments cannot be read this many runs in a row fails the run once (review fix).
+const UNREADABLE_RUNS = 3;
+// WHO IS WAITING (owner, 2026-10-25: "10,747 notifications"). An issue opened by the owner's own account or a bot is
+// work the owner's automation filed (Dream Cycle, dependabot), not a person waiting for an answer: it never pages
+// and gets no acknowledgment. Only a person's issue pages.
+const AUTOMATION_AUTHORS = new Set([OWNER_LOGIN, 'dependabot', 'dependabot[bot]', 'github-actions', 'github-actions[bot]', 'app/dependabot']);
+export const isPersonIssue = (issue) => !AUTOMATION_AUTHORS.has(String(issue?.author?.login || '').toLowerCase())
+  && !issue?.author?.is_bot && !String(issue?.author?.login || '').endsWith('[bot]');
+// ONE morning summary instead of a page per issue per 4 hours: the first run at or after 9 a.m. Eastern sends it,
+// only if a person's issue is still waiting, and at most once per Eastern calendar day.
+const DIGEST_HOUR_ET = 9;
+const etParts = (ms) => Object.fromEntries(new Intl.DateTimeFormat('en-US', { timeZone: 'America/New_York', year: 'numeric',
+  month: '2-digit', day: '2-digit', hour: '2-digit', hour12: false }).formatToParts(new Date(ms)).map((p) => [p.type, p.value]));
+export function digestDue(lastDay, now) {
+  const p = etParts(now); const day = `${p.year}-${p.month}-${p.day}`;
+  return { due: Number(p.hour) % 24 >= DIGEST_HOUR_ET && lastDay !== day, day };
+}
 const STATE_PATH = process.env.ISSUE_WATCH_STATE
   || path.join(os.homedir(), '.claude', 'ruvnet-brain', 'issue-watch-state.json');
 // A compact, always-current snapshot the SessionStart hook surfaces (2026-07-17). ntfy alerts are
@@ -54,31 +72,40 @@ const STATE_PATH = process.env.ISSUE_WATCH_STATE
 const STATUS_PATH = path.join(os.homedir(), '.cache', 'ruvnet-brain', 'open-issues.json');
 const GH_BIN = process.env.GH_BIN || 'gh';
 
-function ghJson(args) {
-  // Retry ONCE on a transient network-shaped failure (2026-07-19, same class as issue-fix's 1am
-  // "TLS handshake timeout" page): 20s of patience absorbs a blip; a second failure still fails
-  // LOUD. Bounded, logged, never silent.
+/** A network-shaped gh failure worth waiting out (2026-07-19 "TLS handshake timeout"; 4.7.21 adds the 1 a.m.
+ *  "error connecting to api.github.com" and GitHub's 502/503/504). Anything else — auth, 404, bad JSON — is real. */
+export function isTransientGhError(text) {
+  return /TLS handshake|unexpected EOF|timeout|ECONNRESET|ETIMEDOUT|EAI_AGAIN|connection refused|temporarily unavailable|error connecting to api\.github\.com|HTTP 50[234]/i.test(String(text || ''));
+}
+/** Seconds to wait before attempts 2 and 3. Bounded: three attempts, then the failure is loud. */
+export const GH_RETRY_DELAYS_S = [10, 30];
+
+/** `gh … --json` with backoff on transient failures. Shared by issue-fix.mjs (one rule, one place). */
+export function ghJsonWithRetry(args, { bin = GH_BIN, who = 'issue-watch' } = {}) {
   let lastErr;
-  for (let attempt = 1; attempt <= 2; attempt++) {
-    const res = spawnSync(GH_BIN, args, { encoding: 'utf8' });
+  for (let attempt = 1; attempt <= GH_RETRY_DELAYS_S.length + 1; attempt++) {
+    const res = spawnSync(bin, args, { encoding: 'utf8' });
     if (res.status === 0) return JSON.parse(res.stdout);
     const err = (res.stderr || res.stdout || '').trim();
     lastErr = new Error(`gh ${args.join(' ')} failed (exit ${res.status}): ${err}`);
-    const transient = /TLS handshake|unexpected EOF|timeout|ECONNRESET|ETIMEDOUT|EAI_AGAIN|connection refused|temporarily unavailable/i.test(err);
-    if (attempt === 1 && transient) {
-      console.error(`issue-watch: transient gh/network failure (${err.slice(0, 90)}) — retrying once in 20s`);
-      spawnSync('sleep', ['20']);
+    if (attempt <= GH_RETRY_DELAYS_S.length && isTransientGhError(err)) {
+      const wait = GH_RETRY_DELAYS_S[attempt - 1];
+      console.error(`${who}: transient gh/network failure (${err.slice(0, 90)}) — retry ${attempt} in ${wait}s`);
+      spawnSync('sleep', [String(wait)]);
       continue;
     }
     break;
   }
   throw lastErr;
 }
+const ghJson = (args) => ghJsonWithRetry(args);
 
 /** Resolve the ntfy topic the same way the rest of the repo does: env, then the machine-wide
  * cache file, then the repo .env — see scripts/notify.sh / scripts/job-heartbeat.sh / scripts/nightly-watchdog.mjs. */
 function resolveTopic() {
-  if (process.env.NTFY_TOPIC) return process.env.NTFY_TOPIC;
+  // Set-but-EMPTY is an explicit opt-out (job-heartbeat.sh's convention): tests use it, and it must never fall
+  // through to the repo's real .env topic (2026-10-25: a unit test read the owner's topic that way).
+  if (process.env.NTFY_TOPIC !== undefined) return process.env.NTFY_TOPIC || null;
   try {
     const t = fs.readFileSync(path.join(os.homedir(), '.cache', 'ruvnet-brain', 'ntfy-topic'), 'utf8').trim();
     if (t) return t;
@@ -92,11 +119,15 @@ function resolveTopic() {
 }
 
 async function pushNtfy(topic, { title, body, priority = 'urgent', tags = 'rotating_light' }) {
+  const base = (process.env.NTFY_URL || 'https://ntfy.sh').replace(/\/+$/, '');
   try {
-    const res = await fetch(`https://ntfy.sh/${topic}`, {
+    // Title is ASCII only: a non-ByteString header makes fetch() throw before sending (4.7.21). Issue titles never
+    // go in the header — they are public, untrusted text — only the body.
+    const res = await fetch(`${base}/${encodeURIComponent(topic)}`, {
       method: 'POST',
-      headers: { Title: title, Priority: priority, Tags: tags },
+      headers: { Title: String(title).replace(/[^\x20-\x7e]+/g, ' ').trim(), Priority: priority, Tags: tags },
       body,
+      signal: AbortSignal.timeout(10_000),
     });
     return res.ok;
   } catch {
@@ -104,8 +135,15 @@ async function pushNtfy(topic, { title, body, priority = 'urgent', tags = 'rotat
   }
 }
 
+/** A missing or unreadable state file is an empty one. That is safe since 4.7.21 (a reset state replayed 17
+ *  "SLA breach" pages in 11 s): every breach due in one run is ONE page, and an acknowledgment already on the
+ *  thread is recognised from the thread itself, never posted twice. */
 function loadState() {
-  try { return JSON.parse(fs.readFileSync(STATE_PATH, 'utf8')); } catch { return {}; }
+  try {
+    const v = JSON.parse(fs.readFileSync(STATE_PATH, 'utf8'));
+    if (v && typeof v === 'object' && !Array.isArray(v)) return v;
+  } catch { /* fall through */ }
+  return {};
 }
 
 function saveState(state) {
@@ -133,28 +171,71 @@ export function judgeIssue(issue, comments, now) {
 
 export async function run({ dryRun = false, now = Date.now(), repo = REPO } = {}) {
   const issues = ghJson(['issue', 'list', '--repo', repo, '--state', 'open', '--json',
-    'number,title,createdAt,comments,updatedAt']);
+    'number,title,createdAt,comments,updatedAt,author']);
 
   const state = loadState();
   const results = [];
   const alertsSent = [];
+  const degraded = [];
+  const dueBreaches = [];
+  const unreadable = []; // person issues unreadable for UNREADABLE_RUNS runs and no other way to judge them: fail once
 
   for (const issue of issues) {
-    const detail = ghJson(['issue', 'view', String(issue.number), '--repo', repo, '--json', 'comments']);
-    const { ageHours, ownerComment, breach } = judgeIssue(issue, detail.comments || [], now);
     const url = `https://github.com/${repo}/issues/${issue.number}`;
     const key = String(issue.number);
-    const last = state[key]?.lastAlertAt ? Date.parse(state[key].lastAlertAt) : null;
-    const dueForAlert = breach && (!last || (now - last) / 3_600_000 >= SLA_HOURS);
+    // One issue whose detail cannot be read does not stop the run (4.7.21). It is judged from the comments the list
+    // call already returned (review fix: before, it was skipped forever and never got its SLA page); with no list
+    // comments either it is skipped, and once it has been unreadable on UNREADABLE_RUNS runs in a row the run fails
+    // ONCE so the heartbeat wrapper pages. Never acknowledged from the list copy (it may be incomplete, and a second
+    // public acknowledgment is worse than none).
+    let detail; let fromList = false;
+    try { detail = ghJson(['issue', 'view', String(issue.number), '--repo', repo, '--json', 'comments']); }
+    catch (err) {
+      degraded.push(issue.number);
+      const prev = state[key] || {};
+      const runs = (Number(prev.unreadableRuns) || 0) + 1;
+      state[key] = { ...prev, unreadableRuns: runs, unreadableSince: prev.unreadableSince || new Date(now).toISOString() };
+      if (Array.isArray(issue.comments)) {
+        detail = { comments: issue.comments }; fromList = true;
+        console.error(`issue-watch: #${issue.number} detail unreadable — judged from the list's comments — ${err.message.slice(0, 160)}`);
+      } else {
+        console.error(`issue-watch: #${issue.number} skipped this run (${runs} in a row) — ${err.message.slice(0, 160)}`);
+        if (isPersonIssue(issue) && runs >= UNREADABLE_RUNS && !prev.unreadableEscalatedAt) {
+          unreadable.push(issue.number);
+          state[key].unreadableEscalatedAt = new Date(now).toISOString();
+        }
+        continue;
+      }
+    }
+    if (!fromList && state[key]?.unreadableRuns) { // readable again: the next unreadable streak counts from zero
+      state[key] = { ...state[key] };
+      for (const f of ['unreadableRuns', 'unreadableSince', 'unreadableEscalatedAt']) delete state[key][f];
+    }
+    const comments = detail.comments || [];
+    const { ageHours, ownerComment, breach } = judgeIssue(issue, comments, now);
+    // A person's issue pages ONCE when it first passes the SLA, never again (it used to repeat every 4 h, forever,
+    // which with ~14 old issues and an hourly run was most of the 10,747 notifications). The morning digest below
+    // carries whatever is still waiting. An existing lastAlertAt means that one page already went out.
+    const person = isPersonIssue(issue);
+    const dueForAlert = person && breach && !state[key]?.lastAlertAt;
     // First sighting → page once, immediately (rule 2 in the header). Delivery-derived like
     // lastAlertAt: state is only written when the push actually went out, so a failed push
     // retries next run instead of burying the sighting.
     const firstSighting = !state[key];
-    // Acknowledgment delivery is not page delivery. Retry each missing channel independently.
-    const needsNewPage = !state[key]?.newAlertAt;
-    const needsAck = !state[key]?.ackAt;
+    // Acknowledgment delivery is not page delivery. Retry each missing channel independently. Neither applies to an
+    // issue the owner's automation opened.
+    // Arrival is paged once, instantly, by .github/workflows/ntfy-alerts.yml (the `issues: opened` event). Paging it
+    // here too sent every new issue twice (review 2026-10-25), so this watcher owns only the SLA page and the digest.
+    const needsNewPage = false;
+    // Our acknowledgment already on the thread (a lost or reset state file) is never posted twice in public.
+    const alreadyAcked = comments.some((c) => c.author?.login === OWNER_LOGIN
+      && /^🤖 Automated acknowledgment/.test(String(c.body || '').trimStart()));
+    if (person && !state[key]?.ackAt && alreadyAcked) {
+      state[key] = { ...(state[key] || {}), ackAt: new Date(now).toISOString(), ackBaselined: true, title: issue.title, url };
+    }
+    const needsAck = person && !fromList && !state[key]?.ackAt;
 
-    results.push({ number: issue.number, title: issue.title, ageHours, ownerComment, breach, dueForAlert, firstSighting, url });
+    results.push({ number: issue.number, title: issue.title, ageHours, ownerComment, breach, dueForAlert, firstSighting, url, person });
 
     if (needsNewPage || needsAck) {
       if (dryRun) {
@@ -191,33 +272,62 @@ export async function run({ dryRun = false, now = Date.now(), repo = REPO } = {}
       }
     }
 
-    if (dueForAlert) {
-      const title = `SLA breach: issue #${issue.number}`;
-      const body = `${issue.title}\nopen ${fmtAge(ageHours)}, no response from ${OWNER_LOGIN}\n${url}`;
-      if (dryRun) {
-        alertsSent.push({ number: issue.number, sent: false, reason: 'dry-run' });
-      } else {
-        const topic = resolveTopic();
-        let sent = false;
-        if (topic) sent = await pushNtfy(topic, { title, body, priority: 'urgent', tags: 'rotating_light,warning' });
-        // DERIVED, not asserted (F5, 2026-07-18): lastAlertAt may only be written when the page was
-        // actually DELIVERED (sent===true). The old line stamped it unconditionally, so a breach whose
-        // push failed (ntfy down, no topic) was suppressed for the whole 4h cooldown — the alert ledger
-        // asserted a delivery it never verified. A failed attempt records itself as failed and the next
-        // hourly run retries; the ledger can no longer claim a page that didn't happen.
-        // Spread-merge, never overwrite: the record may already carry firstSeenAt/newAlertAt from
-        // the first-sighting page above — clobbering them would re-page "new" forever (the exact
-        // state-erasure class that broke issue-fix's comment dedup, 2026-07-24).
-        if (sent) state[key] = { ...(state[key] || {}), lastAlertAt: new Date(now).toISOString(), title: issue.title, url };
-        else state[key] = { ...(state[key] || {}), lastAttemptAt: new Date(now).toISOString(), sent: false, title: issue.title, url };
-        alertsSent.push({ number: issue.number, sent, kind: 'sla-breach', reason: topic ? null : 'no ntfy topic configured' });
+    if (dueForAlert) dueBreaches.push({ issue, ageHours, url, key });
+  }
+
+  // ONE page per run for every breach that is due (4.7.21), oldest first, body bounded to 8 lines. The title keeps
+  // the "SLA breach: issue #N" shape scripts/ntfy-inbox.mjs classifies. DERIVED, not asserted (F5, 2026-07-18):
+  // lastAlertAt is written only for a delivered page; a failed page retries next run as the same single page.
+  // Spread-merge, never overwrite (2026-07-24: clobbering firstSeenAt/newAlertAt re-paged forever).
+  let breachPaged = new Set();
+  if (dueBreaches.length) {
+    dueBreaches.sort((a, b) => b.ageHours - a.ageHours);
+    const [first, ...rest] = dueBreaches;
+    const title = rest.length
+      ? `SLA breach: issue #${first.issue.number} (+${rest.length} more waiting > ${SLA_HOURS}h)`
+      : `SLA breach: issue #${first.issue.number}`;
+    const lines = dueBreaches.slice(0, 8).map((b) => `#${b.issue.number} (${fmtAge(b.ageHours)}) ${String(b.issue.title).slice(0, 70)}`);
+    // Past 8 lines the rest are still NAMED ("…and K more: #a #b"): scripts/ntfy-inbox.mjs files one alert per issue,
+    // so the session banner counts the breach truly (a 17-issue page used to count as 1).
+    const body = `${dueBreaches.length} contributor issue(s) past ${SLA_HOURS}h with no response from ${OWNER_LOGIN}:\n${lines.join('\n')}`
+      + (dueBreaches.length > 8 ? `\n…and ${dueBreaches.length - 8} more: ${dueBreaches.slice(8).map((b) => `#${b.issue.number}`).join(' ')}` : '')
+      + `\nhttps://github.com/${repo}/issues`;
+    if (dryRun) {
+      for (const b of dueBreaches) alertsSent.push({ number: b.issue.number, sent: false, kind: 'sla-breach', reason: 'dry-run' });
+    } else {
+      const topic = resolveTopic();
+      const sent = topic ? await pushNtfy(topic, { title, body, priority: 'urgent', tags: 'rotating_light,warning' }) : false;
+      for (const b of dueBreaches) {
+        if (sent) state[b.key] = { ...(state[b.key] || {}), lastAlertAt: new Date(now).toISOString(), title: b.issue.title, url: b.url };
+        else state[b.key] = { ...(state[b.key] || {}), lastAttemptAt: new Date(now).toISOString(), sent: false, title: b.issue.title, url: b.url };
+        alertsSent.push({ number: b.issue.number, sent, kind: 'sla-breach', reason: topic ? null : 'no ntfy topic configured' });
       }
+      if (sent) breachPaged = new Set(dueBreaches.map((b) => b.issue.number));
+    }
+  }
+
+  // The morning digest: one notification naming every person's issue still unanswered, oldest first.
+  const waiting = results.filter((r) => r.person && r.breach).sort((a, b) => b.ageHours - a.ageHours);
+  const digest = digestDue(state._digest?.day, now);
+  // Everything still waiting was named in this run's breach page: that page IS today's summary.
+  if (waiting.length && digest.due && waiting.every((r) => breachPaged.has(r.number))) {
+    state._digest = { day: digest.day, sentAt: new Date(now).toISOString(), count: waiting.length, viaBreachPage: true };
+  } else if (waiting.length && digest.due) {
+    const title = `Waiting on a reply: ${waiting.length} contributor issue${waiting.length === 1 ? '' : 's'}`;
+    const body = waiting.slice(0, 8).map((r) => `#${r.number} (${fmtAge(r.ageHours)}) ${r.title.slice(0, 70)}`).join('\n')
+      + (waiting.length > 8 ? `\n…and ${waiting.length - 8} more` : '') + `\nhttps://github.com/${repo}/issues`;
+    if (dryRun) alertsSent.push({ kind: 'digest', sent: false, reason: 'dry-run', count: waiting.length });
+    else {
+      const topic = resolveTopic();
+      const sent = topic ? await pushNtfy(topic, { title, body, priority: 'default', tags: 'clipboard' }) : false;
+      if (sent) state._digest = { day: digest.day, sentAt: new Date(now).toISOString(), count: waiting.length };
+      alertsSent.push({ kind: 'digest', sent, count: waiting.length, reason: topic ? null : 'no ntfy topic configured' });
     }
   }
 
   if (!dryRun) saveState(state);
 
-  return { results, alertsSent, checkedAt: new Date(now).toISOString() };
+  return { results, alertsSent, degraded, unreadable, checkedAt: new Date(now).toISOString() };
 }
 
 /**
@@ -353,8 +463,10 @@ async function main() {
       if (r.dueForAlert) {
         const alert = output.alertsSent.find((a) => a.number === r.number && a.kind === 'sla-breach');
         console.log(`   ${dryRun ? '[DRY-RUN] would push ntfy alert' : alert?.sent ? 'ntfy alert pushed' : `ntfy alert NOT sent (${alert?.reason})`}`);
+      } else if (r.breach && !r.person) {
+        console.log('   opened by the owner\'s own automation — never paged');
       } else if (r.breach) {
-        console.log(`   already alerted within the last ${SLA_HOURS}h — not repeating`);
+        console.log('   already paged once — listed in the morning summary until answered');
       }
       console.log('');
     }
@@ -389,7 +501,12 @@ async function main() {
   // heartbeat records the failure and the wrapper's own channel escalates — that duplicate-looking
   // page IS the correct behavior when the primary page provably never left the building.
   const undeliveredAlert = !dryRun && (output.alertsSent || []).some((a) => !a.sent && a.reason !== 'dry-run');
-  process.exit(undeliveredAlert ? 1 : 0);
+  // Skipping one unreadable issue is a degraded run, not a failed one; reading NONE of them is a failure.
+  const allDegraded = output.degraded.length > 0 && output.results.length === 0;
+  if (output.degraded.length) console.error(`issue-watch: ${output.degraded.length} issue(s) not fully readable this run: #${output.degraded.join(', #')}`);
+  const stuck = !dryRun && (output.unreadable || []).length > 0;
+  if (stuck) console.error(`issue-watch: FAILING this run once — a person's issue has been unreadable ${UNREADABLE_RUNS} runs in a row and cannot be judged: #${output.unreadable.join(', #')}`);
+  process.exit(undeliveredAlert || allDegraded || stuck ? 1 : 0);
 }
 
-if (process.argv[1] && import.meta.url === pathToFileURL(path.resolve(process.argv[1])).href) await main();
+if (isDirectInvocation(import.meta.url)) await main();

package/scripts/job-heartbeat.sh

@@ -78,22 +78,41 @@ RUN_ID="$$-$START_EPOCH"
 # SIGKILLed, the watchdog's started-and-never-finished detection had nothing left to see.
 PREV_HB=""
 [ -f "$HB" ] && PREV_HB="$(cat "$HB" 2>/dev/null)"
+# The failing streak and the last page time ride in EVERY receipt, including "running" (4.7.21): a run that is
+# SIGKILLed mid-streak must not erase them, or the next failure would page as a brand-new streak.
+field_str() { printf '%s' "$PREV_HB" | sed -n "s/.*\"$1\":\"\([^\"]*\)\".*/\1/p" | head -1; }
+field_num() { printf '%s' "$PREV_HB" | sed -n "s/.*\"$1\":\([0-9][0-9]*\).*/\1/p" | head -1; }
+prev_state="$(field_str state)"
+prev_since="$(field_str failing_since)"
+prev_page_epoch="$(field_num last_page_epoch)"
+prev_page_at="$(field_str last_page_at)"
+prev_paged="$(printf '%s' "$PREV_HB" | sed -nE 's/.*"paged":("[a-z]*"|true|false).*/\1/p' | head -1)"
+CARRY=""
+[ -n "$prev_page_epoch" ] && CARRY="$CARRY,\"last_page_epoch\":$prev_page_epoch"
+[ -n "$prev_page_at" ] && CARRY="$CARRY,\"last_page_at\":\"$prev_page_at\""
+case "$prev_state" in failed|killed|running)
+  [ -n "$prev_since" ] && CARRY="$CARRY,\"failing_since\":\"$prev_since\""
+  [ -n "$prev_since" ] && [ -n "$prev_paged" ] && CARRY="$CARRY,\"paged\":$prev_paged" ;;
+esac
 
 # Start receipt. If the job vanishes without ever writing an end receipt, THIS is the evidence that
 # it started and never finished — a state the watchdog reports as FAILING, not as silence.
-write_receipt "{\"label\":\"$LABEL\",\"started_at\":\"$STARTED\",\"state\":\"running\",\"pid\":$$,\"run_id\":\"$RUN_ID\",\"command\":\"$(echo "$@" | sed 's/"/\\"/g')\"}"
+write_receipt "{\"label\":\"$LABEL\",\"started_at\":\"$STARTED\",\"state\":\"running\",\"pid\":$$,\"run_id\":\"$RUN_ID\",\"command\":\"$(printf '%s %s' "$1" "${2:-}" | tr -d '"\\\n\r' | cut -c1-200)\"$CARRY}"
 
-notify() { # notify <title> <body> <priority>
+notify() { # notify <title> <body> <priority> — 0 delivered · 1 not delivered · 2 paging disabled (opt-out)
   # NTFY_TOPIC set-but-EMPTY is an explicit opt-out (2026-07-18): unit tests wrap this script around
   # deliberately-failing fixture commands (t-fail, t.bad, exit 7), and the topic-file fallback below
   # meant every test run PAGED Stuart's real phone — 16+ fixture pages in one morning, indistinguishable
   # from real job failures. Tests set NTFY_TOPIC="" and stay silent; production (unset) still falls
-  # through to the topic file.
-  if [ "${NTFY_TOPIC+set}" = "set" ] && [ -z "$NTFY_TOPIC" ]; then return 0; fi
+  # through to the topic file. An opt-out is NOT a delivered page: it returns 2 and the receipt records
+  # "paged":"disabled", never true (4.7.21).
+  if [ "${NTFY_TOPIC+set}" = "set" ] && [ -z "$NTFY_TOPIC" ]; then return 2; fi
   topic="${NTFY_TOPIC:-}"
   [ -z "$topic" ] && [ -f "$HOME/.cache/ruvnet-brain/ntfy-topic" ] && topic="$(cat "$HOME/.cache/ruvnet-brain/ntfy-topic")"
-  [ -z "$topic" ] && return 0
-  curl -sS -m 10 -H "Title: $1" -H "Priority: $3" -H "Tags: rotating_light" -d "$2" "https://ntfy.sh/$topic" >/dev/null 2>&1 || true
+  [ -z "$topic" ] && return 1
+  # --fail: an HTTP error is a failed delivery. The caller records whether the page landed, so a page that
+  # never arrived is retried on the next run instead of muting the whole failing streak (review 2026-10-25).
+  curl -sS --fail -m 10 -H "Title: $1" -H "Priority: $3" -H "Tags: ${4:-rotating_light}" -d "$2" "${NTFY_URL:-https://ntfy.sh}/$topic" >/dev/null 2>&1
 }
 
 # The run_id currently on disk for this label, or empty if the receipt is missing/unreadable/has
@@ -104,6 +123,19 @@ current_run_id() {
 
 finish() {
   code=${FORCED_CODE:-$?}
+  # Stop the run-time timer FIRST, on every exit path (skip, stale writer, normal), so it can never outlive
+  # this wrapper (Opus review 2026-10-25).
+  [ -n "${TIMER:-}" ] && kill "$TIMER" 2>/dev/null
+  timed_out=0
+  if [ -f "$HB.timeout-$RUN_ID" ]; then
+    # Anything the overrunning job left behind (started during its TERM handling, say) goes now.
+    timed_out=1; stop_job
+  fi
+  # Always, on every exit path: the timer may have written the marker after the check above (4.7.21).
+  rm -f "$HB.timeout-$RUN_ID"
+  # An overrun is a failure however the job chose to exit on TERM, including 0 and 75 (the skip code), so this
+  # comes before the skip branch (GPT-6.1 review round 2).
+  [ "$timed_out" -eq 1 ] && [ "$code" -le 128 ] && code=143
   ended="$(ts)"
   dur=$(( $(date +%s) - START_EPOCH ))
   # Exit 75 = SKIP (lock held by a live run). Restore the pre-fire receipt so the live run's evidence
@@ -132,15 +164,52 @@ finish() {
   elif [ "$code" -gt 128 ]; then
     sig=$((code - 128))
     state="killed"; extra=",\"signal\":$sig"
+    [ "$timed_out" -eq 1 ] && extra="$extra,\"timed_out_after_sec\":$MAX_SECONDS"
   else
     state="failed"; extra=""
   fi
-  write_receipt "{\"label\":\"$LABEL\",\"started_at\":\"$STARTED\",\"ended_at\":\"$ended\",\"state\":\"$state\",\"exit_code\":$code,\"duration_sec\":$dur,\"run_id\":\"$RUN_ID\"$extra}"
-  # Gong on failure, immediately — not at the next watchdog sweep. A failing nightly should reach the
-  # phone while it is still tonight's problem.
+  # PAGE ON THE TRANSITION, NOT ON EVERY RUN (owner, 2026-10-25: 10,747 notifications on one topic). A job that
+  # starts failing pages once, urgently, while it is still tonight's problem; while it KEEPS failing it stays
+  # quiet (failing_since carries the first failure; the nightly watchdog's digest still lists it); the run
+  # that passes again sends one low-priority "recovered" note — only if that streak was actually paged.
+  # FLAP DAMPING (4.7.21): a NEW streak within 6 h of the last delivered page is recorded "paged":"suppressed"
+  # instead of paging (4 fail/ok cycles were 8 notifications). A suppressed streak still failing after 6 h pages.
+  # A prior "running" receipt that carries failing_since (a SIGKILLed run) continues its streak.
+  was_failing=0
+  case "$prev_state" in failed|killed) was_failing=1 ;; running) [ -n "$prev_since" ] && was_failing=1 ;; esac
+  # A failing streak counts as already paged only if that page was PROVEN delivered ("paged":true). false (retry),
+  # "suppressed", "disabled", or a receipt from before this field existed are paged (again) when allowed.
+  prev_unpaged=1; [ "$prev_paged" = "true" ] && prev_unpaged=0
+  QUIET_SECONDS=21600
+  now_epoch="$(date +%s)"
+  page_epoch="$prev_page_epoch"; page_at="$prev_page_at"
+  since=""
   if [ "$code" -ne 0 ]; then
-    notify "🔴 SCHEDULED JOB FAILED: $LABEL" "exit $code after ${dur}s — see the job's log. Receipt: $HB" "urgent"
+    if [ "$was_failing" -eq 1 ] && [ -n "$prev_since" ]; then since="$prev_since"; else since="$STARTED"; fi
+    extra="$extra,\"failing_since\":\"$since\""
   fi
+  if [ "$code" -ne 0 ] && { [ "$was_failing" -eq 0 ] || [ "$prev_unpaged" -eq 1 ]; }; then
+    if [ -n "$prev_page_epoch" ] && [ $((now_epoch - prev_page_epoch)) -lt "$QUIET_SECONDS" ] \
+       && ! { [ "$was_failing" -eq 1 ] && [ "$prev_paged" = "false" ]; }; then
+      # Inside the quiet window. (A streak whose own page simply failed to deliver is retried, not damped.)
+      extra="$extra,\"paged\":\"suppressed\""
+    else
+      reason="exit $code after ${dur}s"; [ "$timed_out" -eq 1 ] && reason="stopped after running past its ${MAX_SECONDS}s limit"
+      notify "SCHEDULED JOB FAILED: $LABEL" "$reason — see the job's log. Receipt: $HB. Further failures stay quiet until it recovers." "urgent"
+      rc=$?
+      if [ "$rc" -eq 0 ]; then
+        extra="$extra,\"paged\":true"; page_epoch="$now_epoch"; page_at="$(ts)"
+      elif [ "$rc" -eq 2 ]; then extra="$extra,\"paged\":\"disabled\""
+      else extra="$extra,\"paged\":false"; fi
+    fi
+  elif [ "$code" -ne 0 ]; then
+    extra="$extra,\"paged\":true"
+  elif [ "$was_failing" -eq 1 ] && [ "$prev_paged" = "true" ]; then
+    notify "Recovered: $LABEL" "passed again after failing since ${prev_since:-an earlier run}" "low" "white_check_mark" || true
+  fi
+  [ -n "$page_epoch" ] && extra="$extra,\"last_page_epoch\":$page_epoch"
+  [ -n "$page_at" ] && extra="$extra,\"last_page_at\":\"$page_at\""
+  write_receipt "{\"label\":\"$LABEL\",\"started_at\":\"$STARTED\",\"ended_at\":\"$ended\",\"state\":\"$state\",\"exit_code\":$code,\"duration_sec\":$dur,\"run_id\":\"$RUN_ID\"$extra}"
   exit "$code"
 }
 trap finish EXIT
@@ -148,8 +217,46 @@ trap finish EXIT
 # The exit codes (143 = 128+15 TERM, 130 = 128+2 INT) intentionally reuse the same 128+signal
 # convention `wait` uses for a directly-signalled child, so finish() classifies BOTH as "killed"
 # with the right signal number, whether the signal reached the child directly or via this wrapper.
-trap 'FORCED_CODE=143; kill -TERM "$CHILD" 2>/dev/null; exit 143' TERM
-trap 'FORCED_CODE=130; kill -TERM "$CHILD" 2>/dev/null; exit 130' INT
+# The job stays in this wrapper's process group, so launchd's own cleanup of a dead job (it kills the job's whole
+# group) still covers everything, even if this wrapper is SIGKILLed (Opus review round 3: a separate group escaped it).
+# job_pids: every process the job owns. Under launchd this wrapper leads its group, so that is every group member
+# except this wrapper, its timer, and this wrapper's own helper processes (ps/awk/command substitutions) — which
+# catches processes the job started late or that were reparented. Run any other way (a terminal, a test harness),
+# it falls back to the job's current process tree.
+OWN_GROUP=0; [ "$(ps -o pgid= -p $$ 2>/dev/null | tr -d ' ')" = "$$" ] && OWN_GROUP=1
+job_pids() {
+  if [ "$OWN_GROUP" = 1 ]; then
+    ps -A -o pid= -o ppid= -o pgid= 2>/dev/null | awk -v me=$$ -v t="${TIMER:-0}" -v c="${CHILD:-0}" '
+      { pid[NR]=$1; pp[NR]=$2; pg[NR]=$3 }
+      END { helper[t]=1
+            for (i=1;i<=NR;i++) if (pp[i]==me && pid[i]!=c) helper[pid[i]]=1
+            for (i=1;i<=NR;i++) if (pg[i]==me && pid[i]!=me && !(pid[i] in helper) && !(pp[i] in helper)) print pid[i] }'
+  else
+    tree_of "$CHILD"
+  fi
+}
+tree_of() { kill -0 "$1" 2>/dev/null && printf '%s ' "$1"
+  for k in $(ps -A -o pid= -o ppid= 2>/dev/null | awk -v p="$1" '$2==p {print $1}'); do tree_of "$k"; done; }
+# stop_job: TERM everything the job owns, give it up to 10 s (inside launchd's 20 s exit allowance), then KILL.
+# Every pid seen during the stop is remembered, so a descendant that outlives its parent (and so drops out of the
+# process tree in the fallback mode) still gets the final KILL. The full guarantee, including processes started
+# later, holds when this wrapper leads its group, as under launchd.
+stop_job() {
+  seen="$(job_pids)"; [ -n "$seen" ] && kill -TERM $seen 2>/dev/null
+  i=0; while [ "$i" -lt 10 ]; do
+    now_pids="$(job_pids)"; seen="$seen $now_pids"
+    alive=""; for p in $seen; do kill -0 "$p" 2>/dev/null && alive="$alive $p"; done
+    [ -z "$alive" ] && return 0
+    sleep 1; i=$((i + 1))
+  done
+  for p in $seen $(job_pids); do kill -KILL "$p" 2>/dev/null; done
+  return 0
+}
+trap 'FORCED_CODE=143; [ -n "${TIMER:-}" ] && kill "$TIMER" 2>/dev/null; stop_job; exit 143' TERM
+trap 'FORCED_CODE=130; [ -n "${TIMER:-}" ] && kill "$TIMER" 2>/dev/null; stop_job; exit 130' INT
+# The timer never kills anything itself: it marks the overrun and signals this wrapper, which stops the job.
+OVERRUN=0
+trap 'OVERRUN=1' USR1
 
 # Run the job in the BACKGROUND and `wait` for it — do NOT run it in the foreground.
 # Break-test finding (2026-07-13): a POSIX shell blocked on a FOREGROUND child does not run its trap
@@ -160,4 +267,29 @@ trap 'FORCED_CODE=130; kill -TERM "$CHILD" 2>/dev/null; exit 130' INT
 # ("started and never finished"). Trap for catchable deaths, watchdog for uncatchable ones.
 "$@" &
 CHILD=$!
-wait "$CHILD"
+# RUN-TIME LIMIT (2026-10-25: five jobs finished their work and then never exited, each holding its own
+# next run off for 7-9 hours). After JOB_MAX_SECONDS (default 2 h; a plist may set less) the job is
+# stopped: TERM, then KILL 10 s later. It is recorded as killed and pages once like any other failure.
+MAX_SECONDS="${JOB_MAX_SECONDS:-7200}"
+WRAPPER_PID=$$
+case "$MAX_SECONDS" in ''|*[!0-9]*) MAX_SECONDS=7200 ;; esac
+[ "$MAX_SECONDS" -lt 2 ] && MAX_SECONDS=2
+(
+  # Short sleeps, not one long one: killing this shell then leaves at most a 2 s sleep behind, never a 2 h one
+  # holding the job's output open (review round 2).
+  # A wrapper that is gone (SIGKILLed: launchd then cleans the group itself) ends the timer at the next tick.
+  n=0; while [ "$n" -lt "$MAX_SECONDS" ]; do sleep 2; n=$((n + 2)); kill -0 "$WRAPPER_PID" 2>/dev/null || exit 0; done
+  # Only for OUR child: still alive and still parented by this wrapper. If the PID was reused, do nothing.
+  [ "$(ps -o ppid= -p "$CHILD" 2>/dev/null | tr -d ' ')" = "$WRAPPER_PID" ] || exit 0
+  : > "$HB.timeout-$RUN_ID"
+  kill -USR1 "$WRAPPER_PID" 2>/dev/null
+) </dev/null >/dev/null 2>&1 &
+TIMER=$!
+# `wait` returns early when USR1 arrives; stop the job, then wait again for its real exit status.
+while :; do
+  if [ "$OVERRUN" = 1 ]; then OVERRUN=0; stop_job; fi
+  wait "$CHILD"; RC=$?
+  if [ "$OVERRUN" = 1 ]; then OVERRUN=0; stop_job; continue; fi
+  break
+done
+exit "$RC"

package/scripts/key-canary.mjs

@@ -53,10 +53,14 @@ async function probe(p, key) {
   }
 }
 
+/** true only when notify.sh reports a delivered page (exit 0). A failed or opted-out page (NTFY_TOPIC="") is not
+ *  delivered, so the key's previous state is kept and the transition fires again next run (4.7.21). Titles are
+ *  plain ASCII; the emoji rides in Tags. */
 function notify(title, message, priority, tags) {
   try {
     execFileSync('sh', [path.join(REPO, 'scripts', 'notify.sh'), title, message, priority, tags], { cwd: REPO, stdio: 'ignore', timeout: 15000 });
-  } catch { /* notification failure must not break the canary */ }
+    return true;
+  } catch { return false; /* notification failure must not break the canary */ }
 }
 
 const prev = (() => { try { return JSON.parse(fs.readFileSync(STATE, 'utf8')); } catch { return {}; } })();
@@ -73,13 +77,15 @@ for (const p of PROBES) {
   if (status === 'DEAD') {
     anyDead = true;
     if (NOTIFY && prev[p.env]?.status !== 'DEAD') {
-      notify(`🚨 ${p.provider} API key is DEAD`,
+      const sent = notify(`${p.provider} API key is DEAD`,
         `${p.env} was rejected (401/403) by ${p.provider} just now. Every script using it is silently failing. `
         + `Rotate it: edit ~/Code/openclaw-stack/secrets.env, then run secrets-sync.sh seal.`,
         'urgent', 'rotating_light,key');
+      if (!sent) now[p.env] = prev[p.env] ?? { status: 'unpaged', ts: now[p.env].ts };
     }
   } else if (status === 'alive' && NOTIFY && prev[p.env]?.status === 'DEAD') {
-    notify(`✅ ${p.provider} API key recovered`, `${p.env} is working again.`, 'default', 'white_check_mark,key');
+    const sent = notify(`${p.provider} API key recovered`, `${p.env} is working again.`, 'default', 'white_check_mark,key');
+    if (!sent) now[p.env] = prev[p.env];
   }
 }
 

package/scripts/latency-to-surface.mjs

@@ -43,6 +43,7 @@
 import fs from 'node:fs';
 import os from 'node:os';
 import path from 'node:path';
+import { isDirectInvocation } from '../plugin/scripts/entrypoint.mjs';
 
 const HOME = os.homedir();
 
@@ -206,7 +207,7 @@ export function humanMs(ms) {
 }
 
 // ── CLI ────────────────────────────────────────────────────────────────────────────────────────────
-if (import.meta.url === `file://${process.argv[1]}`) {
+if (isDirectInvocation(import.meta.url)) {
   const { loadOutcomes } = await import('./advocacy-outcomes.mjs');
   const rows = computeLatencies({ outcomes: loadOutcomes() });
   const s = summarize(rows);