ruvnet-brain 4.7.20 → 4.7.21
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/bin/install.mjs +494 -97
- package/kb/download-retry.mjs +93 -21
- package/kb/forge-guard-injection.mjs +277 -0
- package/kb/forge-update.mjs +166 -38
- package/kb/lifecycle-evidence-retention.mjs +151 -14
- package/kb/model-requirements.mjs +4 -1
- package/kb/refresh-run.mjs +175 -27
- package/kb/verify-citation.mjs +15 -2
- package/package.json +3 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/mcp/managed-cli-interface.mjs +16 -8
- package/plugin/scripts/adr-currency-gate.mjs +2 -1
- package/plugin/scripts/advisory-contract.mjs +21 -0
- package/plugin/scripts/advocacy-route.mjs +5 -4
- package/plugin/scripts/agentdb-recall.mjs +15 -5
- package/plugin/scripts/automatic-update.mjs +9 -1
- package/plugin/scripts/brain-confirmation.mjs +2 -1
- package/plugin/scripts/brain-footprint.mjs +72 -11
- package/plugin/scripts/capability-registry.mjs +1 -1
- package/plugin/scripts/capacity-aware-parallel-work.mjs +2 -1
- package/plugin/scripts/codex-overnight-notice.mjs +33 -0
- package/plugin/scripts/completion-claim-evidence.mjs +56 -12
- package/plugin/scripts/continuation-gate.mjs +28 -9
- package/plugin/scripts/decision-gate.mjs +13 -63
- package/plugin/scripts/decision-outcomes.mjs +36 -0
- package/plugin/scripts/decision-recall-gate.mjs +40 -5
- package/plugin/scripts/decision-record.mjs +8 -3
- package/plugin/scripts/degradation-watch.mjs +2 -1
- package/plugin/scripts/developer-update-lock.mjs +172 -33
- package/plugin/scripts/developer-update-owner.mjs +8 -5
- package/plugin/scripts/developer-update-policy.mjs +4 -1
- package/plugin/scripts/developer-update.mjs +155 -53
- package/plugin/scripts/doc-currency.mjs +2 -16
- package/plugin/scripts/duplicate-gate.mjs +2 -1
- package/plugin/scripts/entrypoint.mjs +34 -0
- package/plugin/scripts/footprint-io.mjs +135 -8
- package/plugin/scripts/grok-hook-adapter.mjs +2 -1
- package/plugin/scripts/ground-ruvnet.sh +6 -3
- package/plugin/scripts/grounding-answer.mjs +2 -1
- package/plugin/scripts/grounding-code-projection.mjs +2 -1
- package/plugin/scripts/grounding-substance.mjs +2 -1
- package/plugin/scripts/grounding-turn-gate.mjs +2 -2
- package/plugin/scripts/grounding-turn-mark.mjs +2 -2
- package/plugin/scripts/hook-context-budget.mjs +2 -1
- package/plugin/scripts/hook-input.mjs +2 -2
- package/plugin/scripts/hook-shim.mjs +9 -2
- package/plugin/scripts/host-update.mjs +29 -5
- package/plugin/scripts/identifier-preflight.mjs +2 -1
- package/plugin/scripts/kb-copy-proof.mjs +57 -9
- package/plugin/scripts/learn-capture.mjs +2 -1
- package/plugin/scripts/learning-store.mjs +62 -16
- package/plugin/scripts/lesson-bridge.mjs +2 -2
- package/plugin/scripts/lesson-promote.mjs +83 -22
- package/plugin/scripts/md-stamp.mjs +2 -1
- package/plugin/scripts/native-turn-transcript.mjs +98 -6
- package/plugin/scripts/nightly-scheduler.mjs +10 -5
- package/plugin/scripts/package-recommender.mjs +2 -2
- package/plugin/scripts/project-store-resolver.mjs +19 -0
- package/plugin/scripts/routing-outcome-capture.mjs +2 -1
- package/plugin/scripts/session-snapshot-hook.mjs +24 -4
- package/plugin/scripts/session-start-core.mjs +5 -15
- package/plugin/scripts/session-start-health.mjs +97 -10
- package/plugin/scripts/session-start-issue-alert.mjs +63 -0
- package/plugin/scripts/session-start-update-plane.mjs +15 -1
- package/plugin/scripts/signal-watch.mjs +2 -1
- package/plugin/scripts/spend-guard.mjs +2 -1
- package/plugin/scripts/substitution-wall.mjs +2 -1
- package/plugin/scripts/turn-outcome-capture.mjs +7 -3
- package/plugin/scripts/unprompted-runtime.mjs +20 -3
- package/plugin/scripts/update-apply.mjs +186 -15
- package/plugin/skills/release-proof/scripts/release-proof.mjs +2 -1
- package/plugin/skills/ruvnet-brain/PLAYBOOK.md +2 -2
- package/plugin/skills/ruvnet-brain/SKILL.md +3 -2
- package/scripts/adr-072-completion.mjs +2 -1
- package/scripts/agentdb-context.mjs +2 -1
- package/scripts/agentdb-fleet-doctor.mjs +30 -8
- package/scripts/approved-runtime.mjs +2 -1
- package/scripts/architecture-review-lock.mjs +2 -1
- package/scripts/ascii-drift.mjs +2 -1
- package/scripts/brain-latency-50.mjs +2 -1
- package/scripts/brain-novice-50.mjs +2 -16
- package/scripts/brain-score.mjs +13 -2
- package/scripts/build-bundle.mjs +2 -1
- package/scripts/build-concepts.mjs +2 -1
- package/scripts/candidate-ci-receipt.mjs +2 -1
- package/scripts/candidate-host-evidence.mjs +2 -1
- package/scripts/card-from-source.mjs +2 -1
- package/scripts/ci/warm-brain-models.mjs +2 -1
- package/scripts/claims-verify.mjs +7 -3
- package/scripts/claude-controlled-terminal.mjs +2 -1
- package/scripts/claude-terminal-mod.mjs +2 -1
- package/scripts/code-release-corpus.mjs +2 -1
- package/scripts/codex-hook-trust-reconcile.mjs +67 -1
- package/scripts/codex-hook-trust.mjs +40 -1
- package/scripts/codex-hook-wrapper-history.mjs +2 -1
- package/scripts/codex-host-execution-proof.mjs +2 -1
- package/scripts/completion-claim-replay.mjs +2 -1
- package/scripts/console-engine.mjs +8 -3
- package/scripts/corpus-canary.mjs +2 -1
- package/scripts/corpus-candidate.mjs +2 -2
- package/scripts/corpus-coverage-sidecar.mjs +2 -1
- package/scripts/corpus-dispatch-decision.mjs +2 -1
- package/scripts/corpus-dispatch-receipt.mjs +2 -1
- package/scripts/corpus-next-seed.mjs +2 -1
- package/scripts/corpus-qa.mjs +2 -1
- package/scripts/corpus-reconcile.mjs +2 -2
- package/scripts/corpus-watchdog.mjs +2 -1
- package/scripts/count-chunks.mjs +2 -1
- package/scripts/customer-state-matrix.mjs +2 -1
- package/scripts/derive-passage-content-map.mjs +2 -1
- package/scripts/development-push-check.mjs +2 -1
- package/scripts/dispatch-receipt.mjs +2 -1
- package/scripts/doc-currency.mjs +2 -1
- package/scripts/dream-issue-gate.mjs +2 -1
- package/scripts/dual-host-deliberation.mjs +2 -16
- package/scripts/dual-host-suggest.mjs +2 -16
- package/scripts/eval-brain.mjs +7 -3
- package/scripts/execution-policy.mjs +2 -1
- package/scripts/execution-preflight.mjs +2 -1
- package/scripts/falsify.mjs +2 -1
- package/scripts/fix-metaharness-memretrieve.mjs +2 -1
- package/scripts/fix-workstream.mjs +2 -1
- package/scripts/full-hints.mjs +2 -1
- package/scripts/full-suite-gate.mjs +2 -1
- package/scripts/grok-host.mjs +68 -0
- package/scripts/hook-qualify-core.mjs +3 -1
- package/scripts/hook-qualify.mjs +2 -1
- package/scripts/hook-retirement-check.mjs +2 -1
- package/scripts/host-registry.mjs +2 -1
- package/scripts/independent-review-receipt.mjs +2 -1
- package/scripts/ingest-new-repos.mjs +2 -1
- package/scripts/ingest-repo.mjs +12 -2
- package/scripts/integration-evidence.mjs +2 -1
- package/scripts/issue-fix.mjs +7 -24
- package/scripts/issue-watch.mjs +116 -40
- package/scripts/job-heartbeat.sh +54 -19
- package/scripts/key-canary.mjs +9 -3
- package/scripts/latency-to-surface.mjs +2 -1
- package/scripts/learning-replay-cli.mjs +97 -19
- package/scripts/learning-replay-execution.mjs +2 -0
- package/scripts/learning-replay-fixture.mjs +3 -0
- package/scripts/learning-replay-proof.mjs +14 -13
- package/scripts/learning-replay.mjs +3 -3
- package/scripts/learnings.mjs +2 -1
- package/scripts/lesson-ratify.mjs +88 -40
- package/scripts/loop-checkpoint.mjs +2 -1
- package/scripts/metaharness-receipts.mjs +2 -1
- package/scripts/model-catalog.mjs +2 -1
- package/scripts/model-currency-evidence.mjs +6 -4
- package/scripts/model-currency.mjs +33 -3
- package/scripts/model-native-catalog.mjs +2 -1
- package/scripts/model-router-agent-hook.mjs +2 -1
- package/scripts/model-router-dispatch.mjs +2 -1
- package/scripts/model-router-engine.mjs +2 -1
- package/scripts/model-router-outcome.mjs +2 -1
- package/scripts/model-router-setup.mjs +2 -1
- package/scripts/model-routing-eval.mjs +2 -1
- package/scripts/model-routing-gateway.mjs +2 -1
- package/scripts/model-routing-launchers.mjs +4 -1
- package/scripts/model-terminal-gateway.mjs +2 -1
- package/scripts/model-terminal-launchers.mjs +2 -1
- package/scripts/model-weekly-analyst.mjs +2 -1
- package/scripts/model-weekly-cycle.mjs +2 -1
- package/scripts/model-weekly-qualification.mjs +2 -1
- package/scripts/nightly-gists.sh +26 -20
- package/scripts/nightly-two-run-proof.mjs +2 -1
- package/scripts/nightly-watchdog.mjs +158 -47
- package/scripts/no-silent-substitution.mjs +2 -1
- package/scripts/notify.sh +11 -6
- package/scripts/ntfy-inbox.mjs +41 -18
- package/scripts/onboarding-console.mjs +47 -8
- package/scripts/oracle/abstain-threshold-sweep.mjs +2 -1
- package/scripts/oracle/abstain-trace.mjs +2 -1
- package/scripts/oracle/doc2query-generate.mjs +2 -1
- package/scripts/oracle/doc2query-reach.mjs +2 -1
- package/scripts/oracle/judge-train.mjs +2 -1
- package/scripts/oracle/need-set-split.mjs +2 -1
- package/scripts/oracle/produce-questions.mjs +2 -16
- package/scripts/oracle/repo-recall.mjs +2 -2
- package/scripts/oracle/retrieval-accuracy.mjs +2 -2
- package/scripts/oracle/sona-query-adapter-eval.mjs +2 -1
- package/scripts/oracle/source-tree.mjs +2 -15
- package/scripts/oracle/source-units.mjs +2 -16
- package/scripts/oracle/spike-run.mjs +2 -16
- package/scripts/oracle/validate-labels.mjs +2 -16
- package/scripts/org-repo-count.mjs +2 -1
- package/scripts/package-cards.mjs +2 -1
- package/scripts/prepublication-evidence.mjs +2 -1
- package/scripts/private-overlay.mjs +2 -1
- package/scripts/product-integrity-contract.mjs +2 -1
- package/scripts/protected-release-invocation.mjs +2 -1
- package/scripts/public-verification-abandon.mjs +2 -1
- package/scripts/public-verification-aggregate.mjs +2 -1
- package/scripts/public-verification-finalizer.mjs +2 -1
- package/scripts/public-verification-inputs.mjs +2 -1
- package/scripts/public-verification-lane.mjs +2 -1
- package/scripts/publication-receipt.mjs +2 -1
- package/scripts/published-surface-probe.mjs +2 -1
- package/scripts/qe/card-lane-gate.mjs +2 -16
- package/scripts/qe/session-start-gate.mjs +2 -16
- package/scripts/qe/ux-suite.mjs +2 -1
- package/scripts/qualified-candidate-check.mjs +2 -1
- package/scripts/rebuild-gists-from-receipts.mjs +2 -1
- package/scripts/recommendation-eval.mjs +2 -1
- package/scripts/recommendation-floor.mjs +2 -1
- package/scripts/recommendation-judge-score.mjs +2 -1
- package/scripts/recommendation-real-host-score.mjs +2 -1
- package/scripts/recommendation-real-host.mjs +2 -1
- package/scripts/refresh-capability-only-store.mjs +2 -1
- package/scripts/refresh-model-catalog.mjs +2 -1
- package/scripts/rehearse-corpus-pipeline.mjs +2 -1
- package/scripts/release-authority.mjs +2 -1
- package/scripts/release-evidence-aggregate.mjs +2 -1
- package/scripts/release-payload.mjs +2 -1
- package/scripts/release-projection.mjs +2 -1
- package/scripts/release-proof.mjs +2 -1
- package/scripts/release-qualification-contract.mjs +64 -28
- package/scripts/release-qualification.mjs +8 -1
- package/scripts/release-vector.mjs +2 -1
- package/scripts/restore-local-ingests.mjs +2 -1
- package/scripts/retrieval-canary.mjs +2 -1
- package/scripts/route-cheap.mjs +2 -1
- package/scripts/route-gold-rank.mjs +2 -1
- package/scripts/route-index-memory.mjs +2 -1
- package/scripts/route-latency-warm.mjs +2 -1
- package/scripts/router-utilization.mjs +2 -1
- package/scripts/routing-flywheel.mjs +2 -1
- package/scripts/run-operational-benchmark.mjs +2 -1
- package/scripts/run-operational-benchmark.v3.mjs +2 -1
- package/scripts/rvf-index-audit.mjs +2 -16
- package/scripts/seal-gist-receipt.mjs +2 -1
- package/scripts/security-guidance-codex-compat.mjs +2 -1
- package/scripts/selfcheck.mjs +2 -1
- package/scripts/signal-watch.mjs +2 -1
- package/scripts/source-coverage.mjs +2 -1
- package/scripts/stabilization-receipt.mjs +2 -1
- package/scripts/stamp-sweep.mjs +2 -1
- package/scripts/status-honesty.mjs +2 -1
- package/scripts/sync-commands.mjs +2 -2
- package/scripts/top100-benchmark.mjs +2 -1
- package/scripts/top100-corpus.mjs +2 -1
- package/scripts/user-model-hook-install.mjs +2 -1
- package/scripts/user-model-prompt-hook.mjs +2 -1
- package/scripts/verify-bundle.mjs +3 -1
- package/scripts/verify-model-catalog.mjs +2 -1
- package/scripts/version.mjs +2 -1
- package/scripts/wired-check.mjs +11 -5
- package/scripts/worktree-integrity.mjs +2 -2
|
@@ -8,6 +8,7 @@ import { spawn } from 'node:child_process';
|
|
|
8
8
|
import { fileURLToPath } from 'node:url';
|
|
9
9
|
import { digest, currencyStatus, WEEK_MS } from './model-currency-evidence.mjs';
|
|
10
10
|
import { subscriptionOnlyEnv } from './subscription-hosts.mjs';
|
|
11
|
+
import { isDirectInvocation } from '../plugin/scripts/entrypoint.mjs';
|
|
11
12
|
const DIR = path.dirname(fileURLToPath(import.meta.url));
|
|
12
13
|
const RETRY_MS = 3600000;
|
|
13
14
|
function read(file, limit) {
|
|
@@ -212,7 +213,7 @@ export async function runWeeklyCycle({ routerDir = path.join(os.homedir(), '.cla
|
|
|
212
213
|
try { save('weekly-cycle-last-attempt.json', record); } catch { /* superseded cannot write */ } return record;
|
|
213
214
|
} finally { release(routerDir, token); }
|
|
214
215
|
}
|
|
215
|
-
if (
|
|
216
|
+
if (isDirectInvocation(import.meta.url)) {
|
|
216
217
|
const i = process.argv.indexOf('--router-dir'); const t = process.argv.indexOf('--claim-token'); runWeeklyCycle({ routerDir: i >= 0 ? process.argv[i + 1] : undefined, claimToken: t >= 0 ? process.argv[t + 1] : null }).then((r) => {
|
|
217
218
|
if (r.status !== 'unchanged' || process.argv.includes('--json')) console.log(JSON.stringify(r));
|
|
218
219
|
if (['failed', 'deferred', 'qualification-pending'].includes(r.status)) process.exitCode = 1;
|
|
@@ -6,6 +6,7 @@ import os from 'node:os';
|
|
|
6
6
|
import { randomUUID, randomInt } from 'node:crypto';
|
|
7
7
|
import { fileURLToPath } from 'node:url';
|
|
8
8
|
import { candidateSha256, sha256, promoteRoutingPolicy } from './model-routing-policy-promotion.mjs';
|
|
9
|
+
import { isDirectInvocation } from '../plugin/scripts/entrypoint.mjs';
|
|
9
10
|
|
|
10
11
|
const object = (v) => v !== null && typeof v === 'object' && !Array.isArray(v);
|
|
11
12
|
const same = (a, b) => candidateSha256(a) === candidateSha256(b);
|
|
@@ -359,7 +360,7 @@ async function qualify(ctx) {
|
|
|
359
360
|
return promoted;
|
|
360
361
|
}
|
|
361
362
|
|
|
362
|
-
if (
|
|
363
|
+
if (isDirectInvocation(import.meta.url)) {
|
|
363
364
|
const args = process.argv.slice(2); const value = (flag) => args.includes(flag) ? args[args.indexOf(flag) + 1] : undefined;
|
|
364
365
|
const result = await runWeeklyQualification({ routerDir: value('--router-dir'), semanticReceipt: value('--semantic-receipt'),
|
|
365
366
|
deadline: args.includes('--deadline') ? Number(value('--deadline')) : undefined });
|
package/scripts/nightly-gists.sh
CHANGED
|
@@ -25,29 +25,25 @@ export KB_MODEL_CACHE="${KB_MODEL_CACHE:-/Users/stuartkerr/Code/PowerPlatePulse/
|
|
|
25
25
|
|
|
26
26
|
mkdir -p logs
|
|
27
27
|
LOG="logs/gists-nightly.log"
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
HEARTBEAT_FILE="${JOB_HEARTBEAT_DIR:-$HOME/.cache/ruvnet-brain/heartbeats}/com.ruvnet.brain-gists.heartbeat"
|
|
28
|
+
# Liveness proof is the job-heartbeat.sh wrapper's <label>.json receipt (read by nightly-watchdog.mjs).
|
|
29
|
+
# A 30-second <label>.heartbeat writer used to live here; nothing ever read that file, so it was removed.
|
|
31
30
|
ts() { date '+%Y-%m-%dT%H:%M:%S%z'; }
|
|
32
31
|
log() { printf '[%s] %s\n' "$(ts)" "$1" >>"$LOG"; }
|
|
33
32
|
|
|
34
|
-
# Heartbeat: write a timestamp every 30 seconds to prove the job is still running.
|
|
35
|
-
# Runs in background; killed when the main script exits.
|
|
36
|
-
start_heartbeat() {
|
|
37
|
-
mkdir -p "$(dirname "$HEARTBEAT_FILE")"
|
|
38
|
-
(
|
|
39
|
-
while true; do
|
|
40
|
-
printf '%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$HEARTBEAT_FILE"
|
|
41
|
-
sleep 30
|
|
42
|
-
done
|
|
43
|
-
) &
|
|
44
|
-
HEARTBEAT_PID=$!
|
|
45
|
-
trap "kill $HEARTBEAT_PID 2>/dev/null || true" EXIT
|
|
46
|
-
}
|
|
47
|
-
start_heartbeat
|
|
48
|
-
|
|
49
33
|
log "start"
|
|
50
34
|
|
|
35
|
+
# The plist must NOT say ProcessType=Background. launchd then runs every process in the job under
|
|
36
|
+
# macOS's background policy (efficiency cores only, throttled I/O): measured 2026-10-25 under launchd,
|
|
37
|
+
# one 32-passage bge batch took 111-176 s as Background, 18 s as Standard, 10 s Interactive, 7-13 s
|
|
38
|
+
# from a shell. Eight shards at once pushed a batch past the 15-minute stall budget, so every changed
|
|
39
|
+
# night from 2026-10-08 on aborted as "stalled". Nice 5 plus forge-big's below-normal priority already
|
|
40
|
+
# keep the job polite. Warn loudly if the installed plist regresses.
|
|
41
|
+
PLIST="$HOME/Library/LaunchAgents/com.ruvnet.brain-gists.plist"
|
|
42
|
+
if [ -x /usr/libexec/PlistBuddy ] && [ -f "$PLIST" ] \
|
|
43
|
+
&& [ "$(/usr/libexec/PlistBuddy -c 'Print :ProcessType' "$PLIST" 2>/dev/null)" = "Background" ]; then
|
|
44
|
+
log "WARN: $PLIST has ProcessType=Background — the re-embed will crawl on efficiency cores and trip the stall budget; set it to Standard"
|
|
45
|
+
fi
|
|
46
|
+
|
|
51
47
|
command -v gh >/dev/null 2>&1 || { log "FATAL: gh not on PATH"; exit 1; }
|
|
52
48
|
gh auth status >/dev/null 2>&1 || { log "FATAL: gh not authenticated"; exit 1; }
|
|
53
49
|
|
|
@@ -63,9 +59,19 @@ if ! OUT=$(node scripts/ingest-gists.mjs 2>&1); then
|
|
|
63
59
|
fi
|
|
64
60
|
printf '%s\n' "$OUT" >>"$LOG"
|
|
65
61
|
|
|
62
|
+
# A night whose embed FAILED has already rewritten the passages and the sources receipt, so the next
|
|
63
|
+
# night's ingest says "nothing to do" while the store is still the old one. Without this check that
|
|
64
|
+
# night skipped the embed and exited 0 over a stale store (2026-10-25 20:18: "0 new", store dated
|
|
65
|
+
# 2026-10-07). The store must be at least as new as the passages it is built from.
|
|
66
|
+
PASSAGES="kb/ruv-gists.passages.jsonl"
|
|
67
|
+
STORE="kb/ruv-gists.big.rvf"
|
|
66
68
|
if printf '%s' "$OUT" | grep -q 'nothing to do'; then
|
|
67
|
-
|
|
68
|
-
|
|
69
|
+
if [ -f "$PASSAGES" ] && ! [ "$STORE" -nt "$PASSAGES" ]; then
|
|
70
|
+
log "no new gists, but $STORE is older than $PASSAGES (an earlier embed did not finish) — re-embedding"
|
|
71
|
+
else
|
|
72
|
+
log "no new gists — skipping embed (cost: 0)"
|
|
73
|
+
exit 0
|
|
74
|
+
fi
|
|
69
75
|
fi
|
|
70
76
|
|
|
71
77
|
# 3 — re-embed, only because the corpus actually changed. 8 shards, then one assemble.
|
|
@@ -13,6 +13,7 @@ import { npmInvocation } from './npm-invocation.mjs';
|
|
|
13
13
|
import { packageTreeObservation, observeNpxPackage, validateNpxObservations } from './nightly-package-observation.mjs';
|
|
14
14
|
import { NIGHTLY_LABEL, NIGHTLY_PROOF_LABEL, validateRefreshReceiptEnvelope } from '../plugin/scripts/nightly-scheduler.mjs';
|
|
15
15
|
import { validateCoverageDirectory, validateCoverageLedger } from '../plugin/scripts/coverage-integrity.mjs';
|
|
16
|
+
import { isDirectInvocation } from '../plugin/scripts/entrypoint.mjs';
|
|
16
17
|
|
|
17
18
|
const PUBLIC_KEY = fs.readFileSync(new URL('../keys/ruvnet-brain-signing.pub.pem', import.meta.url), 'utf8');
|
|
18
19
|
|
|
@@ -599,7 +600,7 @@ export async function runNightlyTwoRunProof({ packagePath, bundlePath, out, time
|
|
|
599
600
|
}
|
|
600
601
|
}
|
|
601
602
|
|
|
602
|
-
if ((
|
|
603
|
+
if (isDirectInvocation(import.meta.url)) {
|
|
603
604
|
try {
|
|
604
605
|
const args = parseArgs(process.argv.slice(2));
|
|
605
606
|
const receipt = await runNightlyTwoRunProof({ packagePath: args.package, bundlePath: args.bundle, out: args.out,
|
|
@@ -20,6 +20,8 @@
|
|
|
20
20
|
// NEVER-RAN — loaded, but no receipt has EVER been written (the brain-nightly case)
|
|
21
21
|
// STALE — a receipt exists but is older than the job's schedule allows (it stopped)
|
|
22
22
|
// FAILING — it ran and reported a non-zero exit, or it started and never finished
|
|
23
|
+
// STALLED — running, but its per-shard progress has not moved within its stallMinutes
|
|
24
|
+
// STUCK — still running past its registry maxRunHours with a live wrapper pid (4.7.21)
|
|
23
25
|
// OK — a fresh receipt says it ran and exited 0. The ONLY state that counts as working.
|
|
24
26
|
//
|
|
25
27
|
// Shape confirmed against prior art in the ecosystem: agentic-qe/src/workers/workers/heartbeat-scheduler.ts
|
|
@@ -28,10 +30,20 @@
|
|
|
28
30
|
// Alerts are TRANSITION-ONLY (the discipline proven in kb/brain-alarm.mjs): a state change pages once.
|
|
29
31
|
// Constant redness must never become constant noise, or the gong trains you to ignore it.
|
|
30
32
|
//
|
|
33
|
+
// EXIT CODE = "was the verdict DELIVERED?", not "is every job green?" (4.7.21). A red job is the watchdog doing
|
|
34
|
+
// its job; exiting 1 for it made the heartbeat wrapper page "SCHEDULED JOB FAILED: nightly-watchdog" on top of
|
|
35
|
+
// the real page, every day. Exit 1 now means a page went undelivered or the watchdog itself broke, which the
|
|
36
|
+
// wrapper's own curl page then reports by a second path.
|
|
37
|
+
//
|
|
31
38
|
// Usage:
|
|
32
|
-
// node scripts/nightly-watchdog.mjs
|
|
33
|
-
// node scripts/nightly-watchdog.mjs --
|
|
34
|
-
// node scripts/nightly-watchdog.mjs --
|
|
39
|
+
// node scripts/nightly-watchdog.mjs # verdict; pages transitions; exit 0 when delivered
|
|
40
|
+
// node scripts/nightly-watchdog.mjs --strict # for a human: also exit 1 if ANY job is not OK
|
|
41
|
+
// node scripts/nightly-watchdog.mjs --no-push # judge and print only: no pages, no state write
|
|
42
|
+
// node scripts/nightly-watchdog.mjs --json # machine-readable verdict (implies --no-push)
|
|
43
|
+
// node scripts/nightly-watchdog.mjs --quiet # cron mode: speak only when something is wrong
|
|
44
|
+
// node scripts/nightly-watchdog.mjs --test-page # send one "watchdog delivery test" page; exit 0 only on 2xx
|
|
45
|
+
// Exit codes: 0 delivered (or nothing due) · 1 a page was not delivered, an internal error, or --strict and red ·
|
|
46
|
+
// 2 `launchctl list` was empty or failed (nothing judged, nothing paged, no state written).
|
|
35
47
|
|
|
36
48
|
import fs from 'node:fs';
|
|
37
49
|
import path from 'node:path';
|
|
@@ -40,6 +52,7 @@ import { spawnSync, execFileSync } from 'node:child_process';
|
|
|
40
52
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
41
53
|
import { NIGHTLY_LABEL, schedulerStatus, updateOwnedByAgenticKit } from '../plugin/scripts/nightly-scheduler.mjs';
|
|
42
54
|
import { oldestIncompleteProgress } from '../kb/shard-progress.mjs';
|
|
55
|
+
import { isDirectInvocation } from '../plugin/scripts/entrypoint.mjs';
|
|
43
56
|
|
|
44
57
|
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
45
58
|
const REGISTRY = process.env.WATCHDOG_REGISTRY || path.join(ROOT, 'config', 'scheduled-jobs.json');
|
|
@@ -53,6 +66,10 @@ export const NEVER_RAN = 'NEVER-RAN';
|
|
|
53
66
|
export const STALE = 'STALE';
|
|
54
67
|
export const FAILING = 'FAILING';
|
|
55
68
|
export const STALLED = 'STALLED';
|
|
69
|
+
/** Running longer than the job's registry `maxRunHours`, with its wrapper pid still alive (4.7.21). */
|
|
70
|
+
export const STUCK = 'STUCK';
|
|
71
|
+
/** The watchdog's own label: its receipt reads "running" while it judges, so it never judges itself. */
|
|
72
|
+
const SELF_LABEL = process.env.WATCHDOG_SELF_LABEL || 'com.ruvnet.nightly-watchdog';
|
|
56
73
|
|
|
57
74
|
/**
|
|
58
75
|
* Per-shard progress evidence for a job that declares `progressGlob` + `stallMinutes` in the
|
|
@@ -86,6 +103,16 @@ export function loadedLabels(run = () => spawnSync('launchctl', ['list'], { enco
|
|
|
86
103
|
);
|
|
87
104
|
}
|
|
88
105
|
|
|
106
|
+
/** Is `pid` still OUR wrapper for this label (not a recycled pid)? */
|
|
107
|
+
function wrapperAlive(pid, label) {
|
|
108
|
+
if (!pid) return false;
|
|
109
|
+
try {
|
|
110
|
+
process.kill(pid, 0); // signal 0 = existence check, sends nothing
|
|
111
|
+
const cmd = execFileSync('ps', ['-o', 'command=', '-p', String(pid)], { encoding: 'utf8' });
|
|
112
|
+
return cmd.includes('job-heartbeat.sh') && cmd.includes(label); // guard against pid recycling
|
|
113
|
+
} catch { return false; }
|
|
114
|
+
}
|
|
115
|
+
|
|
89
116
|
/**
|
|
90
117
|
* Judge one job from its receipt. `hb === null` means NO RECEIPT EXISTS — which is NEVER-RAN, never OK.
|
|
91
118
|
* This single line is the whole lesson of 2026-07-13: absence of evidence is not evidence of health.
|
|
@@ -117,6 +144,16 @@ export function judge(job, hb, loaded, now, { root = ROOT } = {}) {
|
|
|
117
144
|
}
|
|
118
145
|
}
|
|
119
146
|
|
|
147
|
+
// STUCK (4.7.21): a job that declares `maxRunHours` and is still "running" past it is stuck even with a live
|
|
148
|
+
// wrapper pid — the 6 h long-run grace below exists for the corpus rebuild, not for a 2-minute job that hung.
|
|
149
|
+
// A gone pid is still the started-and-never-finished FAILING case.
|
|
150
|
+
if (hb.state === 'running' && job.maxRunHours && ageHours > job.maxRunHours) {
|
|
151
|
+
if (wrapperAlive(hb.pid, job.label)) {
|
|
152
|
+
return { state: STUCK, ageHours, detail: `still running after ${ageHours.toFixed(1)}h — its limit is ${job.maxRunHours}h. Wrapper pid ${hb.pid} is alive; the job is not finishing.` };
|
|
153
|
+
}
|
|
154
|
+
return { state: FAILING, ageHours, detail: `started ${ageHours.toFixed(1)}h ago and NEVER FINISHED — receipt says "running" but pid ${hb.pid ?? '?'} is GONE (killed/power loss skipped the trap)` };
|
|
155
|
+
}
|
|
156
|
+
|
|
120
157
|
// Started and never finished? DERIVE it from process liveness, not wall-clock alone (2026-07-19):
|
|
121
158
|
// a full corpus rebuild legitimately runs 12h+ (five changed repos = a long day), and the old
|
|
122
159
|
// ">6h running ⇒ FAILING" rule false-alarmed on exactly that — a 13.5h nightly whose worker was
|
|
@@ -125,15 +162,7 @@ export function judge(job, hb, loaded, now, { root = ROOT } = {}) {
|
|
|
125
162
|
// progress — OK, stated as such. If the pid is GONE while the receipt still says "running", THAT
|
|
126
163
|
// is the real started-and-never-finished (SIGKILL/power loss skipped the trap) — FAILING.
|
|
127
164
|
if (hb.state === 'running' && ageHours > 6) {
|
|
128
|
-
|
|
129
|
-
if (hb.pid) {
|
|
130
|
-
try {
|
|
131
|
-
process.kill(hb.pid, 0); // signal 0 = existence check, sends nothing
|
|
132
|
-
const cmd = execFileSync('ps', ['-o', 'command=', '-p', String(hb.pid)], { encoding: 'utf8' });
|
|
133
|
-
alive = cmd.includes('job-heartbeat.sh') && cmd.includes(job.label); // guard against pid recycling
|
|
134
|
-
} catch { alive = false; }
|
|
135
|
-
}
|
|
136
|
-
if (alive) {
|
|
165
|
+
if (wrapperAlive(hb.pid, job.label)) {
|
|
137
166
|
return { state: OK, ageHours, detail: `LONG RUN in progress — running ${ageHours.toFixed(1)}h, wrapper pid ${hb.pid} verified alive (full rebuilds legitimately take 12h+)` };
|
|
138
167
|
}
|
|
139
168
|
return { state: FAILING, ageHours, detail: `started ${ageHours.toFixed(1)}h ago and NEVER FINISHED — receipt says "running" but pid ${hb.pid ?? '?'} is GONE (killed/power loss skipped the trap)` };
|
|
@@ -190,19 +219,34 @@ export function readLegacyLog(job, root = ROOT) {
|
|
|
190
219
|
} catch { return null; }
|
|
191
220
|
}
|
|
192
221
|
|
|
193
|
-
export function checkAll(now, { registry = REGISTRY, loaded = loadedLabels(), hbDir = HB_DIR, root = ROOT } = {}) {
|
|
222
|
+
export function checkAll(now, { registry = REGISTRY, loaded = loadedLabels(), hbDir = HB_DIR, root = ROOT, self = SELF_LABEL } = {}) {
|
|
194
223
|
const { jobs } = JSON.parse(fs.readFileSync(registry, 'utf8'));
|
|
195
224
|
return jobs.map((job) => {
|
|
196
225
|
const hb = readHeartbeat(job.label, hbDir) || readLegacyLog(job, root);
|
|
226
|
+
// Its own receipt says "running" because this IS that run: judging it is meaningless (it would always read
|
|
227
|
+
// OK, and announce a false recovery). SessionStart and the spend guard judge the watchdog instead (4.7.21).
|
|
228
|
+
if (job.label === self && hb?.state === 'running') {
|
|
229
|
+
return { ...job, state: OK, self: true, detail: 'this run — the watchdog does not judge itself; SessionStart and the spend guard do' };
|
|
230
|
+
}
|
|
197
231
|
const verdict = judge(job, hb, loaded.has(job.label), now, { root });
|
|
198
232
|
if (hb?._tier2 && verdict.state === OK) verdict.detail += ' (via its log — no receipt yet; the next run writes one)';
|
|
199
|
-
|
|
233
|
+
// The heartbeat wrapper already paged (or deliberately damped) this failing streak itself (4.7.21).
|
|
234
|
+
// Only for a TERMINAL receipt: a "running" receipt that never finished carries the old streak's fields forward, but
|
|
235
|
+
// the wrapper never saw that death, so the watchdog must page it.
|
|
236
|
+
const wrapperPaged = verdict.state === FAILING && (hb?.state === 'failed' || hb?.state === 'killed')
|
|
237
|
+
&& !!hb.failing_since && (hb.paged === true || hb.paged === 'suppressed');
|
|
238
|
+
// The wrapper will send its own "Recovered: X" when this job next passes: its receipt carries a failing streak
|
|
239
|
+
// whose page was delivered (job-heartbeat.sh announces recovery exactly then, also after a never-finished run).
|
|
240
|
+
// A watchdog page sent meanwhile (STALE, STUCK, ...) is then recorded as 'both', and its recovery note is left to
|
|
241
|
+
// the wrapper, so one recovery is announced once (4.7.21 review, Fable P2-2).
|
|
242
|
+
const wrapperRecovers = ['failed', 'killed', 'running'].includes(hb?.state) && !!hb.failing_since && hb.paged === true;
|
|
243
|
+
return { ...job, ...verdict, ...(wrapperPaged ? { wrapperPaged: true } : {}), ...(wrapperRecovers ? { wrapperRecovers: true } : {}) };
|
|
200
244
|
});
|
|
201
245
|
}
|
|
202
246
|
|
|
203
247
|
export function productSchedulerVerdict(status) {
|
|
204
248
|
const base = { label: NIGHTLY_LABEL, what: 'Refreshes the installed Brain corpus and converges every detected host',
|
|
205
|
-
schedule: 'daily 03:
|
|
249
|
+
schedule: 'daily 03:30', maxAgeHours: 30, required: true };
|
|
206
250
|
// A 'degraded' adapter whose only defect is a NON-ZERO LAST EXIT is a job that is loaded, verified
|
|
207
251
|
// and FIRED — the exact opposite of MISSING ("not loaded — it can never fire", top of this file).
|
|
208
252
|
// Its truth is the receipt. Measured 2026-09-12 against two FAILED receipts: this projected as
|
|
@@ -240,54 +284,116 @@ const saveState = (s) => { fs.mkdirSync(path.dirname(STATE), { recursive: true }
|
|
|
240
284
|
|
|
241
285
|
/** Page only on a CHANGE of state. Repeating the same alarm nightly is how alarms get ignored. */
|
|
242
286
|
export function transitions(results, prev) {
|
|
243
|
-
return results.filter((r) => (prev[r.label] ?? OK) !== r.state);
|
|
287
|
+
return results.filter((r) => !r.self && (prev[r.label] ?? OK) !== r.state);
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
/** An HTTP header value must be a ByteString. An emoji in `Title` made fetch() throw before sending a byte, so
|
|
291
|
+
* this watchdog never delivered a page (4.7.21). Titles are plain ASCII; the meaning rides in `Tags`. */
|
|
292
|
+
export const asciiTitle = (s) => String(s).replace(/[^\x20-\x7e]+/g, ' ').replace(/\s+/g, ' ').trim().slice(0, 200) || 'RuvNet Brain';
|
|
293
|
+
|
|
294
|
+
/** The topic: NTFY_TOPIC when set (set-but-EMPTY is an explicit opt-out: null), else the owner's topic file. */
|
|
295
|
+
function ntfyTopic() {
|
|
296
|
+
if (process.env.NTFY_TOPIC !== undefined) return process.env.NTFY_TOPIC.trim() || null;
|
|
297
|
+
try { return fs.readFileSync(path.join(os.homedir(), '.cache', 'ruvnet-brain', 'ntfy-topic'), 'utf8').trim() || null; } catch { return null; }
|
|
244
298
|
}
|
|
245
299
|
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
if (!topic) return false;
|
|
300
|
+
/** One POST. Delivered means a 2xx answer — a 429 or 5xx is not a page. */
|
|
301
|
+
async function push(topic, title, body, priority) {
|
|
302
|
+
const base = (process.env.NTFY_URL || 'https://ntfy.sh').replace(/\/+$/, '');
|
|
250
303
|
try {
|
|
251
|
-
await fetch(
|
|
304
|
+
const res = await fetch(`${base}/${encodeURIComponent(topic)}`, {
|
|
252
305
|
method: 'POST',
|
|
253
|
-
headers: { Title: title, Priority: priority, Tags: priority === 'urgent' ? 'rotating_light' : 'white_check_mark' },
|
|
306
|
+
headers: { Title: asciiTitle(title), Priority: priority, Tags: priority === 'urgent' ? 'rotating_light' : 'white_check_mark' },
|
|
254
307
|
body,
|
|
308
|
+
signal: AbortSignal.timeout(10_000),
|
|
255
309
|
});
|
|
256
|
-
return
|
|
310
|
+
return res.ok;
|
|
257
311
|
} catch { return false; }
|
|
258
312
|
}
|
|
259
313
|
|
|
314
|
+
const RETRY_MS = Number(process.env.WATCHDOG_RETRY_MS ?? 30_000);
|
|
315
|
+
const ATTEMPTS = 3;
|
|
316
|
+
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
317
|
+
|
|
318
|
+
/** Send every message, retrying the undelivered ones up to ATTEMPTS times, RETRY_MS apart. Returns the
|
|
319
|
+
* indexes that never landed. */
|
|
320
|
+
async function deliverAll(topic, messages) {
|
|
321
|
+
let pending = messages.map((_, i) => i);
|
|
322
|
+
for (let attempt = 1; attempt <= ATTEMPTS && pending.length; attempt++) {
|
|
323
|
+
if (attempt > 1) await sleep(RETRY_MS);
|
|
324
|
+
const still = [];
|
|
325
|
+
for (const i of pending) if (!(await push(topic, ...messages[i]))) still.push(i);
|
|
326
|
+
pending = still;
|
|
327
|
+
}
|
|
328
|
+
return new Set(pending);
|
|
329
|
+
}
|
|
330
|
+
|
|
260
331
|
async function main() {
|
|
261
|
-
const
|
|
262
|
-
const
|
|
332
|
+
const argv = process.argv.slice(2);
|
|
333
|
+
const json = argv.includes('--json');
|
|
334
|
+
const quiet = argv.includes('--quiet');
|
|
335
|
+
const strict = argv.includes('--strict');
|
|
336
|
+
const noPush = json || argv.includes('--no-push');
|
|
337
|
+
|
|
338
|
+
if (argv.includes('--test-page')) {
|
|
339
|
+
const topic = ntfyTopic();
|
|
340
|
+
if (!topic) { console.error('nightly-watchdog: no ntfy topic configured (NTFY_TOPIC or ~/.cache/ruvnet-brain/ntfy-topic) — nothing sent'); process.exit(1); }
|
|
341
|
+
const failed = await deliverAll(topic, [['Watchdog delivery test', `nightly-watchdog --test-page from ${os.hostname()} at ${new Date().toISOString()}. If you can read this, pages reach you.`, 'default']]);
|
|
342
|
+
console.log(failed.size ? 'nightly-watchdog: test page NOT delivered (no 2xx after 3 attempts)' : 'nightly-watchdog: test page delivered (2xx)');
|
|
343
|
+
process.exit(failed.size ? 1 : 0);
|
|
344
|
+
}
|
|
345
|
+
|
|
346
|
+
// An empty or failed `launchctl list` (launchd busy, sandbox, a broken PATH) would read EVERY job as MISSING and
|
|
347
|
+
// page them all. It is not evidence about any job: say so, page nothing, keep the state (4.7.21).
|
|
348
|
+
const listing = spawnSync('launchctl', ['list'], { encoding: 'utf8' });
|
|
349
|
+
const loaded = loadedLabels(() => (listing.status === 0 ? listing.stdout || '' : ''));
|
|
350
|
+
if (!loaded.size) {
|
|
351
|
+
console.error(`nightly-watchdog: \`launchctl list\` returned no jobs (exit ${listing.status ?? listing.error?.code ?? '?'}) — cannot judge anything; nothing paged, state unchanged`);
|
|
352
|
+
process.exit(2);
|
|
353
|
+
}
|
|
354
|
+
|
|
263
355
|
const brainHome = process.env.RUVNET_BRAIN_HOME || path.join(os.homedir(), '.cache', 'ruvnet-brain');
|
|
264
|
-
const results = [...checkAll(new Date()), ...(brainUpdateOwnedByAgenticKit() ? [] : [productSchedulerVerdict(schedulerStatus({
|
|
356
|
+
const results = [...checkAll(new Date(), { loaded }), ...(brainUpdateOwnedByAgenticKit() ? [] : [productSchedulerVerdict(schedulerStatus({
|
|
265
357
|
brainHome, kbDir: process.env.RUVNET_BRAIN_KB || path.join(brainHome, 'kb'),
|
|
266
358
|
}))])];
|
|
267
359
|
const bad = results.filter((r) => r.state !== OK);
|
|
268
360
|
|
|
269
|
-
|
|
270
|
-
//
|
|
271
|
-
//
|
|
272
|
-
//
|
|
273
|
-
//
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
const
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
361
|
+
// DERIVED, not asserted (F1, 2026-07-18): a transition is only recorded as handled when its page was actually
|
|
362
|
+
// DELIVERED. An undelivered transition keeps its PREVIOUS state, so it re-fires on the next run until a page
|
|
363
|
+
// genuinely lands. A failure the heartbeat wrapper already paged is handled; so is the recovery of a failure the
|
|
364
|
+
// watchdog never paged itself (the wrapper sends that note). The watchdog pages what the wrapper cannot see:
|
|
365
|
+
// MISSING, NEVER-RAN, STALE, STALLED, STUCK, a run that never finished, and their recoveries (4.7.21).
|
|
366
|
+
const topic = noPush ? null : ntfyTopic();
|
|
367
|
+
const optedOut = !noPush && process.env.NTFY_TOPIC !== undefined && !topic;
|
|
368
|
+
let undelivered = new Set();
|
|
369
|
+
if (!noPush && !optedOut) {
|
|
370
|
+
const prev = loadState();
|
|
371
|
+
const pagedBy = { ...(prev._pagedBy || {}) };
|
|
372
|
+
const sends = []; const owners = []; const viaBoth = new Set();
|
|
373
|
+
for (const c of transitions(results, prev)) {
|
|
374
|
+
if (c.wrapperPaged) { pagedBy[c.label] = 'wrapper'; continue; }
|
|
375
|
+
if (c.state === OK) {
|
|
376
|
+
if (pagedBy[c.label] === 'both') { delete pagedBy[c.label]; continue; } // the wrapper's "Recovered" covers it
|
|
377
|
+
if (prev[c.label] === FAILING && pagedBy[c.label] !== 'watchdog') { delete pagedBy[c.label]; continue; }
|
|
378
|
+
sends.push([`${c.label} is healthy again`, c.detail, 'default']); owners.push(c.label);
|
|
379
|
+
} else {
|
|
380
|
+
if (c.wrapperRecovers) viaBoth.add(c.label);
|
|
381
|
+
sends.push([`${c.state}: ${c.label}`, `${c.what}\n\n${c.detail}\n\nschedule: ${c.schedule}`, 'urgent']); owners.push(c.label);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
const failed = topic ? await deliverAll(topic, sends) : new Set(sends.map((_, i) => i));
|
|
385
|
+
undelivered = new Set([...failed].map((i) => owners[i]));
|
|
386
|
+
for (const [i, label] of owners.entries()) {
|
|
387
|
+
if (failed.has(i)) continue;
|
|
388
|
+
if (sends[i][2] === 'urgent') pagedBy[label] = viaBoth.has(label) ? 'both' : 'watchdog'; else delete pagedBy[label];
|
|
389
|
+
}
|
|
390
|
+
const next = Object.fromEntries(results.map((r) => [r.label,
|
|
391
|
+
r.self || undelivered.has(r.label) ? (prev[r.label] ?? OK) : r.state]));
|
|
392
|
+
saveState({ ...next, _pagedBy: pagedBy });
|
|
282
393
|
}
|
|
283
|
-
saveState(Object.fromEntries(results.map((r) => [r.label, undelivered.has(r.label) ? (prev[r.label] ?? OK) : r.state])));
|
|
284
|
-
// Sol amendment to F1: an undelivered page also FAILS this run (exitCode, not exit — the report
|
|
285
|
-
// below still prints). The watchdog's own heartbeat then records the degradation, so "the pager is
|
|
286
|
-
// broken" is itself a paged, visible condition instead of a silent one.
|
|
287
|
-
if (undelivered.size > 0) process.exitCode = 1;
|
|
288
394
|
|
|
289
395
|
if (json) console.log(JSON.stringify({ results, checkedAt: new Date().toISOString() }, null, 2));
|
|
290
|
-
else if (!quiet || bad.length) {
|
|
396
|
+
else if (!quiet || bad.length || undelivered.size) {
|
|
291
397
|
console.log('Scheduled-job watchdog — proof each job RAN, not just that it exists\n');
|
|
292
398
|
for (const r of results) {
|
|
293
399
|
const icon = r.state === OK ? '✅' : '🔴';
|
|
@@ -298,8 +404,13 @@ async function main() {
|
|
|
298
404
|
console.log(bad.length
|
|
299
405
|
? `${bad.length} of ${results.length} job(s) are NOT confirmed working. NEVER-RAN means the schedule has never fired — it does not mean "probably fine".`
|
|
300
406
|
: `All ${results.length} jobs produced a fresh, successful receipt.`);
|
|
407
|
+
if (undelivered.size) console.log(`PAGE NOT DELIVERED for: ${[...undelivered].join(', ')} — it re-fires on the next run.`);
|
|
408
|
+
if (optedOut) console.log('(NTFY_TOPIC="" — paging disabled; no state recorded)');
|
|
301
409
|
}
|
|
302
|
-
|
|
410
|
+
// Sol amendment to F1: an undelivered page FAILS this run, so "the pager is broken" is itself visible.
|
|
411
|
+
process.exit(undelivered.size || (strict && bad.length) ? 1 : 0);
|
|
303
412
|
}
|
|
304
413
|
|
|
305
|
-
if (
|
|
414
|
+
if (isDirectInvocation(import.meta.url)) {
|
|
415
|
+
try { await main(); } catch (e) { console.error(`nightly-watchdog: internal error — ${e?.stack || e}`); process.exit(1); }
|
|
416
|
+
}
|
|
@@ -29,6 +29,7 @@ import fs from 'node:fs';
|
|
|
29
29
|
import path from 'node:path';
|
|
30
30
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
31
31
|
import { substitutionVerdict, inScope, codeOf, message } from '../plugin/scripts/substitution-wall.mjs';
|
|
32
|
+
import { isDirectInvocation } from '../plugin/scripts/entrypoint.mjs';
|
|
32
33
|
|
|
33
34
|
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
34
35
|
|
|
@@ -237,4 +238,4 @@ function main() {
|
|
|
237
238
|
process.exit(1);
|
|
238
239
|
}
|
|
239
240
|
|
|
240
|
-
if (
|
|
241
|
+
if (isDirectInvocation(import.meta.url)) main();
|
package/scripts/notify.sh
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
#!/bin/sh
|
|
2
2
|
# notify.sh "<title>" "<message>" [priority] [tags] — push alert to Stuart's phone via ntfy.sh.
|
|
3
|
-
# Topic from $NTFY_TOPIC or the project .env.
|
|
3
|
+
# Topic from $NTFY_TOPIC or the project .env. Alerting must never break a pipeline (no caller runs under
|
|
4
|
+
# `set -e`), but the exit code now tells the truth so a caller can record delivery (4.7.21):
|
|
5
|
+
# 0 delivered (2xx) · 1 not delivered (network or HTTP error; curl --fail) · 2 paging off (NTFY_TOPIC="" or no topic)
|
|
6
|
+
# NTFY_TOPIC set-but-EMPTY is an explicit opt-out (job-heartbeat.sh's convention): it never falls through to .env.
|
|
4
7
|
DIR=$(dirname "$0")
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
+
if [ "${NTFY_TOPIC+set}" = "set" ]; then TOPIC="$NTFY_TOPIC"
|
|
9
|
+
else TOPIC="$(grep -m1 '^NTFY_TOPIC=' "$DIR/../.env" 2>/dev/null | cut -d= -f2)"; fi
|
|
10
|
+
[ -z "$TOPIC" ] && exit 2
|
|
11
|
+
curl -sS --fail --max-time 10 \
|
|
8
12
|
-H "Title: ${1:-RuvNet Brain}" \
|
|
9
13
|
-H "Priority: ${3:-default}" \
|
|
10
14
|
-H "Tags: ${4:-brain}" \
|
|
11
|
-
-d "${2
|
|
12
|
-
"https://ntfy.sh/$TOPIC" >/dev/null 2>&1 ||
|
|
15
|
+
-d "${2:-(no message)}" \
|
|
16
|
+
"${NTFY_URL:-https://ntfy.sh}/$TOPIC" >/dev/null 2>&1 || exit 1
|
|
17
|
+
exit 0
|
package/scripts/ntfy-inbox.mjs
CHANGED
|
@@ -23,6 +23,7 @@ import { setTimeout as sleep } from 'node:timers/promises';
|
|
|
23
23
|
import { spawn, spawnSync } from 'node:child_process';
|
|
24
24
|
import { fileURLToPath } from 'node:url';
|
|
25
25
|
import { isPersonIssue, judgeIssue } from './issue-watch.mjs';
|
|
26
|
+
import { isDirectInvocation } from '../plugin/scripts/entrypoint.mjs';
|
|
26
27
|
|
|
27
28
|
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
28
29
|
const REPO = 'stuinfla/ruvnet-brain';
|
|
@@ -61,14 +62,27 @@ export function classify(msg) {
|
|
|
61
62
|
let m;
|
|
62
63
|
// job-heartbeat.sh pages "SCHEDULED JOB FAILED: X" / "Recovered: X"; nightly-watchdog.mjs pages "🔴 <STATE>: X" /
|
|
63
64
|
// "✅ X is healthy again". Same cause, same key, so either recovery closes either failure.
|
|
64
|
-
if ((m = /SCHEDULED JOB FAILED: (\S+)/.exec(title)) || (m = /^\W*(?:FAILING|NEVER-RAN|STALE|MISSING|STALLED): (\S+)/.exec(title))) return { kind: 'job', key: `job:${m[1]}`, label: m[1], actionable: true };
|
|
65
|
+
if ((m = /SCHEDULED JOB FAILED: (\S+)/.exec(title)) || (m = /^\W*(?:FAILING|NEVER-RAN|STALE|MISSING|STALLED|STUCK): (\S+)/.exec(title))) return { kind: 'job', key: `job:${m[1]}`, label: m[1], actionable: true };
|
|
65
66
|
if ((m = /Recovered: (\S+)/.exec(title)) || (m = /^\W*(\S+) is healthy again/.exec(title))) return { kind: 'job-recovered', key: `job:${m[1]}`, label: m[1], actionable: false, resolves: true };
|
|
66
67
|
if ((m = /CI \w+: (.+)$/.exec(title))) {
|
|
67
68
|
const b = /branch ([\w./-]{1,100})/.exec(body);
|
|
68
69
|
// Keyed by workflow AND branch: a green main must never close a still-red release branch (review round 4).
|
|
69
70
|
return { kind: 'ci', key: `ci:${m[1].trim()}@${b ? b[1] : 'main'}`, workflow: m[1].trim(), ...(b ? { branch: b[1] } : {}), actionable: true };
|
|
70
71
|
}
|
|
71
|
-
if ((m = /(?:New issue|SLA breach: issue) #(\d+)/.exec(title)))
|
|
72
|
+
if ((m = /(?:New issue|SLA breach: issue) #(\d+)/.exec(title))) {
|
|
73
|
+
// ONE combined SLA page names every breaching issue (scripts/issue-watch.mjs: a "#N (age) title" line each, and
|
|
74
|
+
// "…and K more: #a #b" past 8). Each named issue is its own alert, so a 17-issue breach counts 17, not 1.
|
|
75
|
+
const numbers = [Number(m[1])];
|
|
76
|
+
if (/^SLA breach/.test(title)) {
|
|
77
|
+
for (const line of body.split('\n')) {
|
|
78
|
+
const listed = /^#(\d+) \(/.exec(line);
|
|
79
|
+
if (listed) numbers.push(Number(listed[1]));
|
|
80
|
+
else if (/^…and \d+ more:/.test(line)) for (const x of line.matchAll(/#(\d+)/g)) numbers.push(Number(x[1]));
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
const unique = [...new Set(numbers.filter((n) => Number.isSafeInteger(n) && n > 0))].slice(0, MAX_OPEN_ALERTS);
|
|
84
|
+
return { kind: 'issue', key: `issue:${unique[0]}`, number: unique[0], actionable: true, ...(unique.length > 1 ? { numbers: unique } : {}) };
|
|
85
|
+
}
|
|
72
86
|
if ((m = /New PR #(\d+)/.exec(title))) return { kind: 'pr', key: `pr:${m[1]}`, number: Number(m[1]), actionable: true };
|
|
73
87
|
if (/^Waiting on a reply/.test(title)) return { kind: 'digest', key: 'digest:issues', actionable: false };
|
|
74
88
|
if (/New star/.test(title)) return { kind: 'info', key: `info:${msg.id}`, actionable: false };
|
|
@@ -115,23 +129,32 @@ export function fold(open, messages, { now = Date.now(), resolver = resolvedNow,
|
|
|
115
129
|
for (const msg of messages) {
|
|
116
130
|
if (!msg || typeof msg.id !== 'string' || folded.has(msg.id)) continue;
|
|
117
131
|
folded.add(msg.id);
|
|
118
|
-
const
|
|
132
|
+
const classified = classify(msg);
|
|
119
133
|
const at = new Date((msg.time || now / 1000) * 1000).toISOString();
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
const
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
continue;
|
|
134
|
+
// A combined SLA page becomes one alert per issue it names; the first keeps the message id, the rest get id-N.
|
|
135
|
+
const parts = classified.numbers ? classified.numbers.map((n, i) => {
|
|
136
|
+
const one = { ...classified, key: `issue:${n}`, number: n, alertId: i === 0 ? msg.id : `${msg.id}-${n}` };
|
|
137
|
+
delete one.numbers;
|
|
138
|
+
return one;
|
|
139
|
+
}) : [classified];
|
|
140
|
+
for (const c of parts) {
|
|
141
|
+
if (c.resolves) {
|
|
142
|
+
const a = byKey.get(c.key);
|
|
143
|
+
if (a && !a.closedAt && Date.parse(at) >= Date.parse(a.lastAt) && resolver(a)) { a.closedAt = at; a.closedBy = 'recovered notice (confirmed)'; a.closedByMsg = msg.id; }
|
|
144
|
+
continue;
|
|
145
|
+
}
|
|
146
|
+
if (!c.actionable) continue;
|
|
147
|
+
const existing = byKey.get(c.key);
|
|
148
|
+
if (existing && !existing.closedAt) {
|
|
149
|
+
// lastAt only moves forward: a delayed older message must not make a newer failure look resolved.
|
|
150
|
+
existing.count += 1;
|
|
151
|
+
if (Date.parse(at) > Date.parse(existing.lastAt)) { existing.lastAt = at; existing.lastTitle = msg.title; }
|
|
152
|
+
existing.msgIds = [...(existing.msgIds || []), msg.id].slice(-10);
|
|
153
|
+
continue;
|
|
154
|
+
}
|
|
155
|
+
const { alertId, ...fields } = c;
|
|
156
|
+
byKey.set(c.key, { id: alertId || msg.id, ...fields, firstAt: at, lastAt: at, count: 1, title: msg.title || '', body: String(msg.message || '').slice(0, 200), msgIds: [msg.id], attempts: 0 });
|
|
133
157
|
}
|
|
134
|
-
byKey.set(c.key, { id: msg.id, ...c, firstAt: at, lastAt: at, count: 1, title: msg.title || '', body: String(msg.message || '').slice(0, 200), msgIds: [msg.id], attempts: 0 });
|
|
135
158
|
}
|
|
136
159
|
const byOldestCheck = [...byKey.values()].sort((x, y) => Date.parse(x.lastCheckedAt || 0) - Date.parse(y.lastCheckedAt || 0));
|
|
137
160
|
for (const a of byOldestCheck) {
|
|
@@ -506,7 +529,7 @@ export function formatOpen(alerts) {
|
|
|
506
529
|
+ `${a.count > 1 ? ` (×${a.count}, since ${clean(a.firstAt, 30)})` : ''}\n ${clean(String(a.body || '').split('\n')[0], 160)}`).join('\n') + tail;
|
|
507
530
|
}
|
|
508
531
|
|
|
509
|
-
if (
|
|
532
|
+
if (isDirectInvocation(import.meta.url)) {
|
|
510
533
|
const args = process.argv.slice(2);
|
|
511
534
|
try {
|
|
512
535
|
if (args[0] === '--open') {
|