ruvnet-brain 4.7.14 → 4.7.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -9,7 +9,7 @@ Created: 2026-06-29 22:36:38 EDT
9
9
 
10
10
  ### Discover the stack. Read the source. Build with confidence.
11
11
 
12
- [![RuvNet Brain version 4.6.2 — README updated 2026-10-10 EDT](https://img.shields.io/badge/version_4.7.14-updated_2026--10--10_EDT-1E90FF?style=for-the-badge&labelColor=0757BA)](plugin/.claude-plugin/plugin.json)
12
+ [![RuvNet Brain version 4.6.2 — README updated 2026-10-10 EDT](https://img.shields.io/badge/version_4.7.16-updated_2026--10--10_EDT-1E90FF?style=for-the-badge&labelColor=0757BA)](plugin/.claude-plugin/plugin.json)
13
13
 
14
14
  **A source-grounded companion for Claude Code and Codex that helps you find and use Reuven Cohen's (rUv's) RuvNet ecosystem.**
15
15
 
@@ -255,7 +255,7 @@ uses the protected release workflow; a consumer update downloads an available ge
255
255
  does not rebuild upstream knowledge. Check `--doctor`, `SOURCE.json` and refresh receipts
256
256
  before treating an installation as current.
257
257
 
258
- The **4.6.0 published-corpus baseline** is **203 public stores · 164,402 public source chunks**.
258
+ The **4.6.0 published-corpus baseline** is **203 public stores · 164,863 public source chunks**.
259
259
  Those are bundle census values, not a claim that every upstream repository is fresh today.
260
260
  Later corpus publication may advance the knowledge generation independently of code changes.
261
261
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ruvnet-brain",
3
- "version": "4.7.14",
3
+ "version": "4.7.16",
4
4
  "description": "One-command installer for RuvNet Brain — a portable, source-grounded brain over rUv's RuvNet building blocks, delivered as a Claude Code plugin so Claude uses the stack instead of fighting it.",
5
5
  "type": "module",
6
6
  "bin": {
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "ruvnet-brain",
3
3
  "description": "RuvNet brain transplant for Claude Code — grounds every RuvNet decision in real source across 77 rUv repositories, prefers Ruflo / RuVector-RVF / AgentDB over training-prior defaults (pgvector, Pinecone, hand-rolled cosine), and can pull in any RuvNet repo on demand. Ships a UserPromptSubmit retrieve-and-inject grounding hook and a PreToolUse write gate that refuses ungrounded rUv-product code until search_ruvnet has been consulted (ADR-0012 / ADR-067).",
4
- "version": "4.7.14",
4
+ "version": "4.7.16",
5
5
  "author": {
6
6
  "name": "Stuart Kerr"
7
7
  },
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ruvnet-brain",
3
- "version": "4.7.14",
3
+ "version": "4.7.16",
4
4
  "description": "Source-grounded RuvNet knowledge, lifecycle enforcement, and learning for Codex.",
5
5
  "author": {
6
6
  "name": "Stuart Kerr"
@@ -293,11 +293,17 @@ export function eligibleRepositoryStanding(row) {
293
293
  if (status === 'CURRENT') return carry === undefined && failure === undefined ? 'shipped' : null;
294
294
  if (status === 'STALE' && carry && typeof carry === 'object' && failure === undefined) {
295
295
  const carried = String(carry.carriedSourceCommit || '').toLowerCase();
296
+ const upstream = String(row.upstream?.sha || '').toLowerCase();
297
+ // Two honest shapes. A refresh that FAILED: upstream was read, differs from the carried commit, and is named
298
+ // as missedUpstream. An upstream nobody could READ tonight (transient, source-coverage.mjs): the row is pinned
299
+ // at the carried commit, says `upstreamUnavailable`, and missedUpstream is null -- unknown, never invented.
300
+ const missed = String(carry.missedUpstream || '').toLowerCase();
301
+ const upstreamKnown = HEX40.test(missed) && missed === upstream && carried !== missed;
302
+ const upstreamUnread = carry.missedUpstream === null && row.upstreamUnavailable === true && row.upstreamUnavailableKind === 'transient'
303
+ && carry.reason === 'transient: upstream unreadable' && carried === upstream;
296
304
  const shipped = typeof carry.reason === 'string' && carry.reason.length > 0
297
305
  && HEX40.test(carried) && carried === String(row.artifact?.sourceCommit || '').toLowerCase()
298
- && HEX40.test(String(carry.missedUpstream || '').toLowerCase())
299
- && String(carry.missedUpstream).toLowerCase() === String(row.upstream?.sha || '').toLowerCase()
300
- && carried !== String(carry.missedUpstream).toLowerCase()
306
+ && (upstreamKnown || upstreamUnread)
301
307
  && isAttemptCount(carry.attempts) && isIsoOrNull(carry.carriedCommittedAt)
302
308
  && row.artifact?.bytesVerified === true && row.artifact?.passagesPresent === true;
303
309
  return shipped ? 'shipped' : null;
@@ -286,7 +286,7 @@ export async function synchronizePlugins(run, dryRun, notes, { home, prefix, sco
286
286
  if (JSON.stringify(flags) !== JSON.stringify(pluginFlags(writable, home))) throw Error('Codex plugin enablement changed');
287
287
  return { before, after, steps, provider };
288
288
  }
289
- function knowledge(run, dryRun, runId, { brainHome, root, node }) {
289
+ export function knowledge(run, dryRun, runId, { brainHome, root, node }) {
290
290
  const kb = path.join(brainHome, 'kb'), updater = path.join(kb, 'forge-update.mjs');
291
291
  if (!exists(updater)) throw Error('existing Brain KB updater absent; fresh install refused');
292
292
  const file = path.join(brainHome, `nightly-suite-corpus-${runId}.json`);
@@ -299,7 +299,10 @@ function knowledge(run, dryRun, runId, { brainHome, root, node }) {
299
299
  }
300
300
  const before = check();
301
301
  if (before.currencyVerdict === 'CURRENT' || before.currencyVerdict === 'REFUSED') return { before, after: before, state: before.currencyVerdict === 'CURRENT' ? 'current' : 'ahead-preserved' };
302
- if (before.currencyVerdict !== 'UPDATE_AVAILABLE') throw Error(`corpus currency unverified: ${before.currencyVerdict}`);
302
+ // UNKNOWN is what a tree installed from a code release reports (no corpus generation stamp): forge-update
303
+ // documents it as apply-allowed, exactly like UPDATE_AVAILABLE, and the check after the apply must reach
304
+ // CURRENT or REFUSED. Throwing here failed every scheduled night until someone ran --update by hand.
305
+ if (before.currencyVerdict !== 'UPDATE_AVAILABLE' && before.currencyVerdict !== 'UNKNOWN') throw Error(`corpus currency unverified: ${before.currencyVerdict}`);
303
306
  if (dryRun) return { before, state: 'update-available' };
304
307
  const installer = path.join(root, 'ruvnet-brain/bin/install.mjs');
305
308
  if (!exists(installer)) throw Error('existing Brain installer absent');
@@ -46,6 +46,7 @@ import fs from 'node:fs';
46
46
  import os from 'node:os';
47
47
  import path from 'node:path';
48
48
  import { spawnSync } from 'node:child_process';
49
+ import { retryTransientResult } from './transient-gh.mjs';
49
50
  import { fileURLToPath, pathToFileURL } from 'node:url';
50
51
 
51
52
  const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
@@ -190,9 +191,12 @@ const semverDescending = (left, right) => {
190
191
  return b[0] - a[0] || b[1] - a[1] || b[2] - a[2];
191
192
  };
192
193
 
194
+ // Every read here is a `gh` read of public release metadata or assets; a transient failure (rate limit, 5xx,
195
+ // a reset mid-download of the ~555 MB archive) is retried, bounded, before it can fail the dispatcher, the
196
+ // identity job or the publisher (scripts/transient-gh.mjs). A refused aggregate is an answer and is never retried.
193
197
  function defaultGh(args) {
194
- const result = spawnSync(process.env.RUVNET_GH_COMMAND || 'gh', args,
195
- { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, timeout: 15 * 60_000 });
198
+ const result = retryTransientResult(() => spawnSync(process.env.RUVNET_GH_COMMAND || 'gh', args,
199
+ { encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, timeout: 15 * 60_000 }));
196
200
  if (result.error || result.status !== 0) {
197
201
  throw new Error(`gh ${args.slice(0, 3).join(' ')} failed: ${String(result.error?.message || result.stderr || `exit ${result.status}`).trim().slice(0, 400)}`);
198
202
  }
@@ -47,6 +47,7 @@ import { addPrivateStore, resolveRuntime, resultFromRefreshReceipt, runInstaller
47
47
  import { CANARY_VERDICT_KIND, CORPUS_TAG_PATTERN, REQUIRED_CANARY_CHECKS } from './corpus-promotion.mjs';
48
48
  import { inventoryFootprint } from '../plugin/scripts/brain-footprint.mjs';
49
49
  import { eligibleRepositoryStanding } from '../plugin/scripts/coverage-integrity.mjs';
50
+ import { TRANSIENT_HTTP, retryTransientAsync } from './transient-gh.mjs';
50
51
 
51
52
  export const PUBLIC_API = 'https://api.github.com';
52
53
  export const FRESHNESS_LIMIT_MS = 48 * 3_600_000;
@@ -151,9 +152,11 @@ export function storeFreshness({ after, before = null, now }) {
151
152
  const upstream = row.upstream?.sha || null;
152
153
  const built = row.artifact?.sourceCommit || null;
153
154
  if (!upstream || !built) continue;
155
+ // Named either way: a refresh that failed (upstream read, differs) or an upstream nobody could read tonight
156
+ // (pinned at the carried commit, `upstreamUnavailable`). Both are carried with a record the validator accepts.
157
+ if (row.kind === 'repository' && row.carry && eligibleRepositoryStanding(row) === 'shipped') { carried.push(row.artifact?.store || row.key); continue; }
154
158
  if (upstream !== built) {
155
- if (row.kind === 'repository' && row.carry && eligibleRepositoryStanding(row) === 'shipped') carried.push(row.artifact?.store || row.key);
156
- else stale.push(row.artifact?.store || row.key);
159
+ stale.push(row.artifact?.store || row.key);
157
160
  continue;
158
161
  }
159
162
  if (before && priorCommit.get(row.key) !== upstream) moved += 1;
@@ -285,15 +288,18 @@ function runLogged(command, args, { env, cwd, timeoutMs, log }) {
285
288
  return result;
286
289
  }
287
290
 
288
- /** Phase 1: the approved runtime, installed exactly as `npx ruvnet-brain@<version>` would. */
289
- export function installCustomer({ approvedVersion, work, home, log = () => {} }) {
291
+ /**
292
+ * Phase 1: the approved runtime, installed exactly as `npx ruvnet-brain@<version>` would. The registry read is
293
+ * retried on a transient failure (bounded) so a registry blip reads as weather, never as "the customer refused".
294
+ */
295
+ export async function installCustomer({ approvedVersion, work, home, log = () => {}, retryDelayMs = 30_000 }) {
290
296
  const env = customerEnv({ home, work });
291
297
  fs.mkdirSync(env.TMPDIR, { recursive: true });
292
298
  const prefix = path.join(work, 'npx');
293
299
  fs.mkdirSync(prefix, { recursive: true });
294
300
  const npm = path.join(path.dirname(process.execPath), process.platform === 'win32' ? 'npm.cmd' : 'npm');
295
- runLogged(npm, ['install', '--prefix', prefix, '--no-audit', '--no-fund', '--no-save', `ruvnet-brain@${approvedVersion}`],
296
- { env, cwd: prefix, timeoutMs: 600_000, log });
301
+ await retryTransientAsync(() => runLogged(npm, ['install', '--prefix', prefix, '--no-audit', '--no-fund', '--no-save', `ruvnet-brain@${approvedVersion}`],
302
+ { env, cwd: prefix, timeoutMs: 600_000, log }), { baseDelayMs: retryDelayMs, onRetry: ({ attempt }) => log(`--- npm install failed transiently (attempt ${attempt}); retrying`) });
297
303
  const installer = path.join(prefix, 'node_modules', 'ruvnet-brain', 'bin', 'install.mjs');
298
304
  const installed = readJson(path.join(prefix, 'node_modules', 'ruvnet-brain', 'package.json')).version;
299
305
  if (installed !== approvedVersion) throw new Error(`npm served ruvnet-brain ${installed}, not the approved ${approvedVersion}`);
@@ -316,8 +322,8 @@ function runUpdater({ kbDir, env, resultFile }) {
316
322
  }
317
323
 
318
324
  // An anonymous GitHub API read from a shared runner address is sometimes refused (403/429) or fails (5xx)
319
- // for a moment; one such blip must not fail the night. The same call is retried after a short wait.
320
- const TRANSIENT_HTTP = new Set([403, 429, 500, 502, 503, 504]);
325
+ // for a moment; one such blip must not fail the night. The same call is retried after a short wait. The
326
+ // status set is the pipeline's one definition (scripts/transient-gh.mjs TRANSIENT_HTTP), not a private copy.
321
327
  async function fetchJson(url, what, { retryDelayMs = 15_000, maxAttempts = 4 } = {}) {
322
328
  for (let attempt = 1; ; attempt += 1) {
323
329
  let status = null;
@@ -19,11 +19,31 @@
19
19
  // the new workflow passes (--approved-tag, --no-change-out, --stage-candidate, --promote-staged), so the nightly stands down rather than
20
20
  // let new workflow text drive old scripts.
21
21
 
22
+ //
23
+ // THE RETRY SLOT (2026-10-20). Measured on 38 nights: 24 failed, most on one transient blip, and nothing
24
+ // retried until the next calendar day. The dispatcher now also fires at RETRY_CRON (11:17 UTC, four hours
25
+ // after the night's first slot) and asks decideRetry(): re-dispatch ONLY when every corpus run of the
26
+ // night so far ended `failed` (or reached no outcome), none is still running, none ended published /
27
+ // no-change / superseded / canary-rejected (those are answers, not weather), and fewer than
28
+ // MAX_CORPUS_RUNS_PER_NIGHT runs exist -- so one night gets at most one retry, and a deterministic
29
+ // failure fails closed a second time rather than looping. The kill switch and the approved-runtime
30
+ // resolution still run after it, exactly as on the first slot. Double publication is impossible by
31
+ // construction: a failed run promoted nothing, every generation is content-addressed, and release.mjs
32
+ // refuses a tag that already exists.
33
+
22
34
  import fs from 'node:fs';
23
35
  import path from 'node:path';
24
36
  import { pathToFileURL } from 'node:url';
25
37
 
26
38
  export const MIN_PIPELINE_VERSION = '4.3.38'; // sync-version-ignore: a fixed pipeline floor, not the current version
39
+ /** The dispatcher's second schedule entry; corpus-nightly-dispatch.yml's `on.schedule` must carry exactly this line. */
40
+ export const RETRY_CRON = '17 11 * * *';
41
+ /** How far back the retry slot looks for the night's runs: past the 07:17 UTC first slot, never into yesterday. */
42
+ export const RETRY_WINDOW_MS = 10 * 3_600_000;
43
+ export const MAX_CORPUS_RUNS_PER_NIGHT = 2;
44
+ // `unknown` (every job green, no outcome) is retried only when the run itself did NOT conclude success: a green
45
+ // run that reached no outcome is a workflow defect the terminal-outcome job already fails loudly, not weather.
46
+ const retryable = (run) => run.outcome === 'failed' || (run.outcome === 'unknown' && run.conclusion !== 'success');
27
47
  const SEMVER = /^(\d+)\.(\d+)\.(\d+)$/;
28
48
  const HEX40 = /^[0-9a-f]{40}$/;
29
49
  const CODE_TAG = /^v\d+\.\d+\.\d+$/;
@@ -82,6 +102,21 @@ export function decide({ nightlyVar, resolution, minPipelineVersion = MIN_PIPELI
82
102
  return { verdict: 'arm', reason: `building the corpus at approved runtime ${tag} @ ${sourceSha}`, tag, version, sourceSha };
83
103
  }
84
104
 
105
+ /**
106
+ * The retry slot's one decision. `runs`: the night's protected-release corpus runs so far
107
+ * (scripts/corpus-watchdog.mjs gatherCorpusRuns, each with `outcome` from classifyCorpusRun).
108
+ * Returns { verdict: 'retry' | 'stand-down', reason }.
109
+ */
110
+ export function decideRetry({ runs = [], maxRuns = MAX_CORPUS_RUNS_PER_NIGHT } = {}) {
111
+ if (!runs.length) return { verdict: 'stand-down', reason: 'the first slot dispatched no corpus run tonight (it stood down or failed); the watchdog judges that, a retry would not' };
112
+ const running = runs.find((run) => run.outcome === 'in-progress');
113
+ if (running) return { verdict: 'stand-down', reason: `corpus run ${running.id} is still in progress` };
114
+ const settled = runs.find((run) => !retryable(run));
115
+ if (settled) return { verdict: 'stand-down', reason: `corpus run ${settled.id} ended ${settled.outcome}; that is the night's answer, not weather` };
116
+ if (runs.length >= maxRuns) return { verdict: 'stand-down', reason: `${runs.length} corpus run(s) already failed tonight; the bound is ${maxRuns} per night, so this is not retried again` };
117
+ return { verdict: 'retry', reason: `corpus run ${runs.map((run) => run.id).join(', ')} failed and nothing published; dispatching the night's one retry` };
118
+ }
119
+
85
120
  /** approved-runtime.mjs --resolve's exit status + stdout (+ stderr, naming a refused release) -> a resolution. */
86
121
  export function resolutionFrom({ status, stdout, stderr = '' }) {
87
122
  if (String(status) === '3') return { status: 'not-yet-verified' };
@@ -107,12 +142,31 @@ const arg = (argv, name) => {
107
142
  * CLI: --nightly <value> --resolve-status <n|skipped> --release <file> [--resolve-stderr <file>]
108
143
  * --github-output <file>. Exit 1 only on `fail`.
109
144
  * --is-off <value> prints true|false (lets the workflow skip the ~555 MB resolve when the switch is off)
145
+ * --retry-slot --repo <owner/name> [--now ISO] --github-output <file>
146
+ * writes retry=true|false and retry_reason; exit 1 only when run history could not be read
110
147
  */
111
- export function main(argv = process.argv.slice(2), { stdout = process.stdout } = {}) {
148
+ export async function main(argv = process.argv.slice(2), { stdout = process.stdout } = {}) {
112
149
  if (argv.includes('--is-off')) {
113
150
  stdout.write(`${killSwitchOff(arg(argv, '--is-off'))}\n`);
114
151
  return 0;
115
152
  }
153
+ if (argv.includes('--retry-slot')) {
154
+ const { classifyCorpusRun, gatherCorpusRuns } = await import('./corpus-watchdog.mjs');
155
+ const now = arg(argv, '--now') ? Date.parse(arg(argv, '--now')) : Date.now();
156
+ const repo = arg(argv, '--repo') || process.env.GITHUB_REPOSITORY;
157
+ let decision;
158
+ try {
159
+ const runs = gatherCorpusRuns({ repo, since: now - RETRY_WINDOW_MS }).map((run) => ({ ...run, outcome: classifyCorpusRun(run) }));
160
+ decision = decideRetry({ runs });
161
+ } catch (error) {
162
+ stdout.write(`::error::corpus-nightly-dispatch retry slot could not read the night's runs: ${error.message}\n`);
163
+ return 1;
164
+ }
165
+ const githubOutput = arg(argv, '--github-output');
166
+ if (githubOutput) fs.appendFileSync(path.resolve(githubOutput), `retry=${decision.verdict === 'retry'}\nretry_reason=${decision.reason.replace(/\n/g, ' ')}\n`);
167
+ stdout.write(`::notice::corpus-nightly-dispatch retry slot ${decision.verdict}: ${decision.reason}\n`);
168
+ return 0;
169
+ }
116
170
  const nightlyVar = arg(argv, '--nightly') ?? '';
117
171
  const status = arg(argv, '--resolve-status');
118
172
  const releaseFile = arg(argv, '--release');
@@ -134,5 +188,5 @@ export function main(argv = process.argv.slice(2), { stdout = process.stdout } =
134
188
  }
135
189
 
136
190
  if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) {
137
- process.exitCode = main();
191
+ main().then((code) => { process.exitCode = code; });
138
192
  }
@@ -61,6 +61,7 @@ import path from 'node:path';
61
61
  import { spawnSync } from 'node:child_process';
62
62
  import { fileURLToPath, pathToFileURL } from 'node:url';
63
63
  import { COVERAGE_ASSET, COVERAGE_RECEIPT_ASSET, bindCoverageToReceipt, verifyCoverageSidecar } from './corpus-coverage-sidecar.mjs';
64
+ import { retryTransientResult } from './transient-gh.mjs';
64
65
 
65
66
  const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
66
67
  const HEX64 = /^[0-9a-f]{64}$/;
@@ -76,7 +77,10 @@ const REQUIRED_ASSETS = [ARCHIVE_ASSET, SIGNATURE_ASSET, RECEIPT_ASSET, RECALL_A
76
77
  /** ADR-0091 D4: judge at most this many of the newest generations, then use the committed bootstrap. */
77
78
  export const SEARCH_BOUND = 5;
78
79
 
79
- const defaultRun = (command, args, options) => spawnSync(command, args, { encoding: 'utf8', ...options });
80
+ // Every `gh` call below is a read (release list, release view, small-asset download). A transient failure is
81
+ // retried, bounded, before it is recorded as `indeterminate` (scripts/transient-gh.mjs): an unretried blip here
82
+ // used to send the whole night back to the committed bootstrap seed -- a full rebuild dressed as a quiet night.
83
+ const defaultRun = (command, args, options) => retryTransientResult(() => spawnSync(command, args, { encoding: 'utf8', ...options }));
80
84
 
81
85
  function ghJson(run, args) {
82
86
  const result = run('gh', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
@@ -25,6 +25,7 @@ import { storeRoot } from '../kb/store-root.mjs';
25
25
  import { captureGistSources } from './gist-receipts.mjs';
26
26
  import { projectSourceStore, RUNTIME_LEDGER_KIND } from './rvf-generation.mjs';
27
27
  import { compareKnowledgeInputs, fromCoverage as knowledgeFromCoverage, fromSeed as knowledgeFromSeed } from './knowledge-input-digest.mjs';
28
+ import { defaultBaseDelayMs } from './transient-gh.mjs';
28
29
 
29
30
  export { rebuildCorpusAggregates };
30
31
 
@@ -352,8 +353,10 @@ export async function acquireSealedGeneration({ maxAttempts = 3, assetsDir = nul
352
353
  const settled = await build(observation, storeOutcomes);
353
354
  const remaining = planReconciliation({ coverage: settled, ledger: currentLedger(), assetsDir })
354
355
  .filter((item) => !recorded(item.store));
356
+ // A repository row the shipped validator accepts WITH a carry -- a transiently unreadable source pinned at its
357
+ // seed commit (source-coverage.mjs) -- is resolved for this generation, exactly like a recorded store outcome.
355
358
  const unresolved = settled.rows.filter((row) => row.disposition === 'eligible' && row.status !== 'CURRENT'
356
- && !(row.kind === 'repository' && recorded(row.artifact?.store)));
359
+ && !(row.kind === 'repository' && (recorded(row.artifact?.store) || (row.carry && eligibleRepositoryStanding(row) === 'shipped'))));
357
360
  attempts.push({ attempt, plan, ...reconciliation, ...pruning, ...aggregates,
358
361
  remainingArtifacts: remaining.length, unresolvedSources: unresolved.length });
359
362
  if (!remaining.length && !unresolved.length) {
@@ -635,11 +638,13 @@ export function dispositionForFailedStore({ assetsDir, ledger, item, attempts, r
635
638
 
636
639
  // Only a TRANSIENT failure is retried, exactly once, in a fresh directory: `<store>-retry1`. The first
637
640
  // attempt's directory is never reused -- a clone or a half-written worker output from the failed
638
- // attempt must not be mistaken for the retry's own (the pre-D5 path collided).
641
+ // attempt must not be mistaken for the retry's own (the pre-D5 path collided). The retry waits
642
+ // TRANSIENT_RETRY_DELAY_MS first: an immediate re-clone into the same reset connection learns nothing.
639
643
  export function workerRootFor(workspace, store, retry) {
640
644
  return path.join(workspace, 'workers', retry === 0 ? store : `${store}-retry${retry}`);
641
645
  }
642
646
  export const MAX_TRANSIENT_RETRIES = 1;
647
+ export const TRANSIENT_RETRY_DELAY_MS = 30_000;
643
648
 
644
649
  export async function executeReconciliation({
645
650
  plan,
@@ -651,6 +656,8 @@ export async function executeReconciliation({
651
656
  signal,
652
657
  priorCoverage = null,
653
658
  log = (line) => console.log(line),
659
+ retryDelayMs = defaultBaseDelayMs(TRANSIENT_RETRY_DELAY_MS),
660
+ sleep = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); }),
654
661
  }) {
655
662
  if (!Array.isArray(plan) || !Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 10) {
656
663
  fail('reconciliation plan or worker concurrency is invalid');
@@ -750,8 +757,10 @@ export async function executeReconciliation({
750
757
  lastError = error;
751
758
  const retrying = isRetryable(error) && retry < MAX_TRANSIENT_RETRIES;
752
759
  log(`[corpus-reconcile] ${item.store}: attempt ${attempts} failed -- ${error.message}`
753
- + (retrying ? '; retrying once in a fresh worker directory' : '; not retried'));
760
+ + (retrying ? `; retrying once in a fresh worker directory after ${retryDelayMs / 1000}s` : '; not retried'));
754
761
  if (!retrying) break;
762
+ await sleep(retryDelayMs);
763
+ if (controller.signal.aborted) throw abortError(controller.signal);
755
764
  }
756
765
  }
757
766
  return { ok: false, attempts, error: lastError };
@@ -871,7 +880,33 @@ async function observeSourceOnly({ owner, assetsDir }) {
871
880
  const externalFile = path.join(assets, 'external-sources.json');
872
881
  const policy = fs.existsSync(externalFile) ? readJson(externalFile, 'external source policy') : { sources: [] };
873
882
  if (!Array.isArray(policy.sources)) fail('external source policy has no sources array');
874
- return observeSourceUniverse({ owner, externalSources: policy.sources });
883
+ const ledgerFile = path.join(assets, 'RVF-GENERATIONS.json');
884
+ const ledger = fs.existsSync(ledgerFile) ? readJson(ledgerFile, 'RVF generation ledger') : { stores: {} };
885
+ return observeSourceUniverse({ owner, externalSources: policy.sources,
886
+ carriedCommits: transientCarryCommits({ ledger, priorCoverage: readPriorCoverage(assets) }) });
887
+ }
888
+
889
+ /**
890
+ * The seed's own commits, per store: what an external source unreadable tonight is pinned at (one night, STALE
891
+ * with a carry) instead of failing the whole observation. Read from the seed ledger, never invented -- and a store
892
+ * the SEED's sealed coverage already shows carried this way is left out, so the second consecutive night fails
893
+ * for that source exactly as before rather than carrying it forever.
894
+ */
895
+ export function transientCarryCommits({ ledger, priorCoverage = null }) {
896
+ // No sealed coverage to read history from (the bootstrap lineage, or an invalid sidecar) = no carry at all:
897
+ // the allowance exists only where the previous night can prove it was not already spent.
898
+ if (!Array.isArray(priorCoverage?.rows)) return {};
899
+ // ANY carried seed row spends it -- a failed refresh last night and an unreadable upstream tonight must not
900
+ // alternate forever either.
901
+ const alreadyCarried = new Set(priorCoverage.rows
902
+ .filter((row) => row?.kind === 'repository' && row.carry)
903
+ .map((row) => String(row.artifact?.store || '').toLowerCase()));
904
+ return Object.fromEntries(Object.entries(ledger?.stores || {})
905
+ .filter(([store, row]) => HEX40.test(String(row?.sourceCommit || '').toLowerCase()) && !alreadyCarried.has(store.toLowerCase()))
906
+ .map(([store, row]) => {
907
+ const sourceCommit = String(row.sourceCommit).toLowerCase();
908
+ return [store.toLowerCase(), { sourceCommit, committedAt: carriedCommittedAt({ priorCoverage, store, sourceCommit }) }];
909
+ }));
875
910
  }
876
911
 
877
912
  export async function acquireCorpusGeneration({ owner = 'ruvnet', assetsDir, workspaceDir,
@@ -22,6 +22,10 @@
22
22
  // WARNING a superseded or degraded night, a stand-down tonight, an unknown outcome, a long run
23
23
  // GREEN none of the above
24
24
  //
25
+ // THE FIRST LINE IS THE NIGHT'S VERDICT (2026-10-20, summarizeNight): what the night did -- published,
26
+ // no-change, degraded with the stores named, failed with the stage named, superseded, canary-rejected, still
27
+ // running, stood down -- in Eastern time, result first, so the owner never learns of a break from a page.
28
+ //
25
29
  // Exit code is the contract (same as scripts/github-health-watch.mjs): 0 = GREEN or WARNING,
26
30
  // 1 = RED or the evidence could not be gathered. corpus-watchdog.yml runs on a genuine `schedule:`,
27
31
  // so its red conclusion fires ntfy-alerts.yml's workflow_run listener — no secret needed here.
@@ -43,6 +47,7 @@ import { CODE_TAG_PATTERN, isCorpusReleaseTag } from './release-channel-kind.mjs
43
47
  import { AGGREGATE_ASSET, SIGNING_PUBLIC_KEY_FILE } from './approved-runtime.mjs';
44
48
  import { COVERAGE_ASSET, COVERAGE_RECEIPT_ASSET, verifyCoverageSidecar } from './corpus-coverage-sidecar.mjs';
45
49
  import { parseCorpusGeneration } from './corpus-promotion.mjs';
50
+ import { retryTransientResult } from './transient-gh.mjs';
46
51
 
47
52
  const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
48
53
  export const RED = 'RED';
@@ -255,7 +260,9 @@ export function judgeCorpusHealth(input, now) {
255
260
  // ── Gathering (read-only gh). Everything above is pure; everything below talks to GitHub. ──────────
256
261
 
257
262
  function defaultGh(args) {
258
- const result = spawnSync(process.env.RUVNET_GH_COMMAND || 'gh', args, { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 5 * 60_000 });
263
+ // A transient gh failure is retried, bounded (scripts/transient-gh.mjs); the judge must not page RED on weather.
264
+ const result = retryTransientResult(() => spawnSync(process.env.RUVNET_GH_COMMAND || 'gh', args,
265
+ { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 5 * 60_000 }));
259
266
  if (result.error || result.status !== 0) {
260
267
  throw new Error(`gh ${args.slice(0, 3).join(' ')} failed: ${String(result.error?.message || result.stderr || `exit ${result.status}`).trim().slice(0, 300)}`);
261
268
  }
@@ -320,17 +327,25 @@ function corpusRunEvidence({ gh, repo, run, scratch }) {
320
327
  }
321
328
  }
322
329
 
330
+ /**
331
+ * The protected-release corpus runs created since `since` (ms), each with its outcome evidence. Shared with the
332
+ * dispatcher's retry slot (scripts/corpus-dispatch-decision.mjs --retry-slot), so both read one definition of
333
+ * "tonight's runs" and one classifier.
334
+ */
335
+ export function gatherCorpusRuns({ repo, since, gh = defaultGh, scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'corpus-watchdog-')) }) {
336
+ return JSON.parse(gh(['run', 'list', '--repo', repo, '--workflow', 'protected-release.yml', '--limit', '60',
337
+ '--json', 'databaseId,attempt,displayTitle,status,conclusion,createdAt,updatedAt']))
338
+ .filter((run) => CORPUS_RUN_TITLE.test(String(run.displayTitle || '')) && (ms(run.createdAt) ?? 0) >= since)
339
+ .map((run) => ({ id: run.databaseId, title: run.displayTitle, status: run.status, conclusion: run.conclusion,
340
+ createdAt: run.createdAt, updatedAt: run.updatedAt, ...corpusRunEvidence({ gh, repo, run, scratch }) }));
341
+ }
342
+
323
343
  export async function gatherCorpusHealthInput({ repo, now, gh = defaultGh, root = ROOT, scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'corpus-watchdog-')) }) {
324
344
  const at = now.getTime();
325
- const recent = (iso) => (ms(iso) ?? 0) >= at - 3 * 24 * HOUR;
326
345
  const dispatcherRuns = JSON.parse(gh(['run', 'list', '--repo', repo, '--workflow', 'corpus-nightly-dispatch.yml', '--limit', '30',
327
346
  '--json', 'databaseId,attempt,event,status,conclusion,createdAt,updatedAt']))
328
347
  .map((run) => ({ id: run.databaseId, attempt: run.attempt, event: run.event, status: run.status, conclusion: run.conclusion, createdAt: run.createdAt }));
329
- const corpusRuns = JSON.parse(gh(['run', 'list', '--repo', repo, '--workflow', 'protected-release.yml', '--limit', '60',
330
- '--json', 'databaseId,attempt,displayTitle,status,conclusion,createdAt,updatedAt']))
331
- .filter((run) => CORPUS_RUN_TITLE.test(String(run.displayTitle || '')) && recent(run.createdAt))
332
- .map((run) => ({ id: run.databaseId, title: run.displayTitle, status: run.status, conclusion: run.conclusion,
333
- createdAt: run.createdAt, updatedAt: run.updatedAt, ...corpusRunEvidence({ gh, repo, run, scratch }) }));
348
+ const corpusRuns = gatherCorpusRuns({ repo, since: at - 3 * 24 * HOUR, gh, scratch });
334
349
  const releases = JSON.parse(gh(['release', 'list', '--repo', repo, '--limit', '100', '--json', 'tagName,publishedAt,isDraft,isPrerelease']));
335
350
  // PROMOTED only: a staged candidate (and a bootstrap seed) is a prerelease no customer receives.
336
351
  const corpusReleases = releases.filter((row) => !row.isDraft && !row.isPrerelease && isCorpusReleaseTag(row.tagName))
@@ -364,8 +379,72 @@ export async function gatherCorpusHealthInput({ repo, now, gh = defaultGh, root
364
379
  return { corpusReleases, corpusRuns, dispatcherRuns, codeRelease, deferral, latestRelease };
365
380
  }
366
381
 
382
+ /** A time the owner reads: local Eastern, 12-hour, named zone -- never UTC or "Z". */
383
+ export function formatEastern(when) {
384
+ const t = when instanceof Date ? when.getTime() : ms(when);
385
+ if (t === null || !Number.isFinite(t)) return 'an unknown time';
386
+ const parts = Object.fromEntries(new Intl.DateTimeFormat('en-US', { timeZone: 'America/New_York', month: 'short', day: 'numeric',
387
+ hour: 'numeric', minute: '2-digit', hour12: true }).formatToParts(new Date(t)).map((part) => [part.type, part.value]));
388
+ return `${parts.hour}:${parts.minute} ${parts.dayPeriod.toLowerCase().replace(/^(a|p)m$/, '$1.m.')} ET ${parts.month} ${parts.day}`;
389
+ }
390
+
391
+ /** The job the night died in, named as the owner knows the stage, from the terminal outcome record or the job list. */
392
+ const STAGE_NAMES = { identity: 'authorize (corpus-identity)', prepare: 'prepare/seal (seal-exact-corpus-candidate)',
393
+ authorize: 'authenticate (corpus-authorize)', publish: 'publisher (corpus-publish)', canary: 'customer canary', promote: 'promote' };
394
+ const JOB_NAMES = { 'authorize-corpus-generation': 'identity', 'seal-exact-corpus-candidate': 'prepare', 'authenticate-corpus-preparation-provenance': 'authorize',
395
+ 'protected-corpus-publisher': 'publish', 'customer-canary': 'canary', 'promote-canaried-corpus': 'promote' };
396
+ export function failedStage(run) {
397
+ const recorded = Object.entries(run?.outcomeRecord?.jobs || {}).find(([, result]) => result === 'failure')?.[0];
398
+ const listed = (run?.jobs || []).find((job) => job.conclusion === 'failure' && JOB_NAMES[job.name]);
399
+ const stage = recorded || (listed ? JOB_NAMES[listed.name] : null);
400
+ return stage ? STAGE_NAMES[stage] || stage : `an unnamed job (conclusion ${run?.conclusion || 'unknown'})`;
401
+ }
402
+
403
+ /**
404
+ * THE ONE LINE the owner reads first: what the night did, result first, in Eastern time. `input` is
405
+ * gatherCorpusHealthInput's; the night is the newest corpus run of the last 24h and every retry of it.
406
+ * Never learned from a page: the watchdog writes this before ntfy ever fires.
407
+ */
408
+ export function summarizeNight(input, now) {
409
+ const at = now instanceof Date ? now.getTime() : Number(now);
410
+ const inWindow = (iso) => { const t = ms(iso); return t !== null && t <= at && at - t <= TONIGHT_WINDOW_MS; };
411
+ const runs = (input?.corpusRuns || []).filter((run) => inWindow(run.createdAt))
412
+ .map((run) => ({ ...run, outcome: run.outcome ?? classifyCorpusRun(run) })).sort((a, b) => ms(b.createdAt) - ms(a.createdAt));
413
+ const dispatcher = (input?.dispatcherRuns || []).filter((d) => inWindow(d.createdAt)).sort((a, b) => ms(b.createdAt) - ms(a.createdAt))[0];
414
+ const latest = input?.latestRelease;
415
+ const attempts = runs.length > 1 ? ` (attempt ${runs.length} of the night; ${runs.length - 1} retried)` : '';
416
+ const keep = latest ? `customers keep ${latest.tag} (${hours(Math.max(0, at - (ms(latest.generation || latest.publishedAt) ?? at)))} old)` : 'no promoted generation is observed';
417
+ if (!runs.length) {
418
+ if (!dispatcher) return { outcome: 'no-run', line: `night: no corpus run in the last 24h (no dispatcher run either); ${keep}` };
419
+ if (dispatcher.status !== 'completed') return { outcome: 'dispatching', line: `night: dispatcher run ${dispatcher.id} is ${dispatcher.status} (started ${formatEastern(dispatcher.createdAt)})` };
420
+ if (dispatcher.conclusion !== 'success') return { outcome: 'dispatcher-failed', line: `night: dispatcher run ${dispatcher.id} concluded ${dispatcher.conclusion} at ${formatEastern(dispatcher.createdAt)}, nothing was dispatched; ${keep}` };
421
+ return { outcome: 'stood-down', line: `night: stood down at ${formatEastern(dispatcher.createdAt)} (CORPUS_NIGHTLY off, or no install-verified runtime); ${keep}` };
422
+ }
423
+ const [run] = runs;
424
+ const when = formatEastern(run.updatedAt || run.createdAt);
425
+ const degraded = input?.deferral?.observable && input.deferral.degraded && latest && input.deferral.newestTag === latest.tag
426
+ && (input.deferral.degraded.carried.length || input.deferral.degraded.missing.length) ? input.deferral.degraded : null;
427
+ switch (run.outcome) {
428
+ case 'published': {
429
+ const tag = latest && (ms(latest.publishedAt) ?? 0) >= (ms(run.createdAt) ?? 0) ? latest.tag : 'a new generation';
430
+ if (degraded) {
431
+ return { outcome: 'degraded', line: `night: degraded -- published ${tag} at ${when}${attempts} with ${degraded.carried.length} carried `
432
+ + `(${degraded.carried.join(', ') || 'none'}) and ${degraded.missing.length} missing (${degraded.missing.join(', ') || 'none'}), each named in CORPUS-COVERAGE.json` };
433
+ }
434
+ return { outcome: 'published', line: `night: published ${tag} at ${when}${attempts}` };
435
+ }
436
+ case 'no-change': return { outcome: 'no-change', line: `night: no-change at ${when}${attempts} -- every knowledge input equals the promoted generation; ${keep}` };
437
+ case 'superseded': return { outcome: 'superseded', line: `night: superseded at ${when} -- a newer code release landed; the next night builds at it; ${keep}` };
438
+ case 'canary-rejected': return { outcome: 'canary-rejected', line: `night: canary-rejected at ${when}${attempts} -- a clean customer install could not apply the candidate (GLOBAL: not retried); ${keep}` };
439
+ case 'in-progress': return { outcome: 'in-progress', line: `night: run ${run.id} still in progress, started ${formatEastern(run.createdAt)}${attempts}` };
440
+ case 'failed': return { outcome: 'failed', line: `night: failed in ${failedStage(run)} at ${when}${attempts}; ${keep}` };
441
+ default: return { outcome: 'unknown', line: `night: run ${run.id} succeeded at ${when} but reached no outcome${attempts}; ${keep}` };
442
+ }
443
+ }
444
+
367
445
  export function renderReport(result, now) {
368
- const lines = [`corpus-watchdog ${result.verdict} at ${now.toISOString()}`];
446
+ const lines = [`corpus-watchdog ${result.verdict} -- ${result.night?.line || `judged ${formatEastern(now)}`}`,
447
+ ` judged ${formatEastern(now)} (${now.toISOString()})`];
369
448
  for (const f of result.findings) lines.push(` [${f.level}] ${f.check}: ${f.detail}`);
370
449
  return `${lines.join('\n')}\n`;
371
450
  }
@@ -380,7 +459,8 @@ async function main(argv = process.argv.slice(2)) {
380
459
  }
381
460
  let result;
382
461
  try {
383
- result = judgeCorpusHealth(await gatherCorpusHealthInput({ repo, now }), now);
462
+ const input = await gatherCorpusHealthInput({ repo, now });
463
+ result = { ...judgeCorpusHealth(input, now), night: summarizeNight(input, now) };
384
464
  } catch (error) {
385
465
  // Could not observe = could not prove health. Absence of evidence is failure.
386
466
  process.stderr.write(`corpus-watchdog RED: evidence could not be gathered: ${error.message}\n`);
@@ -34,6 +34,7 @@ import { canonicalJson, digest, validateGistAggregateReceipt } from './coverage-
34
34
  import { promoteArtifactSet } from '../kb/incremental-refresh.mjs';
35
35
  import { writeRvfGeneration } from './rvf-generation.mjs';
36
36
  import { fetchGistGitSnapshot } from './gist-git-transport.mjs';
37
+ import { isTransientText, retryTransientAsync } from './transient-gh.mjs';
37
38
 
38
39
  const DEFAULT_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
39
40
  const TEXT_EXT = new Set(['.md', '.markdown', '.txt', '.rst']);
@@ -402,9 +403,15 @@ function reusableCachedGist(cached, stub) {
402
403
  return 'reuse';
403
404
  }
404
405
 
406
+ // The Git snapshot is one `ls-remote` + one bare clone per gist against gist.github.com. A transient transport
407
+ // failure (reset, timeout, "could not read Username", or the gist moving mid-capture) is retried, bounded, so
408
+ // one blip on one of ~180 gists cannot fail the aggregate and with it the night. A gist that is gone
409
+ // ("repository not found") is an answer and is thrown at once.
410
+ const transientGistSnapshot = (error) => isTransientText(error?.message) || /moved while Git snapshot was captured/.test(String(error?.message || ''));
411
+
405
412
  export async function captureGistSources({ observation, cache = null, fetchDetail = null,
406
413
  fetchRaw = defaultFetchRaw, fetchGitSnapshot = fetchGistGitSnapshot, signal,
407
- now = () => new Date().toISOString() } = {}) {
414
+ now = () => new Date().toISOString(), snapshotRetryDelayMs = 15_000 } = {}) {
408
415
  const stubs = observation?.gists?.rows;
409
416
  if (!Array.isArray(stubs) || stubs.some(({ id }) => !HEX_GIST.test(String(id || '')))
410
417
  || new Set(stubs.map(({ id }) => id)).size !== stubs.length) {
@@ -420,7 +427,8 @@ export async function captureGistSources({ observation, cache = null, fetchDetai
420
427
  // The production/no-key route never spends one REST detail request per gist and never trusts
421
428
  // a body-free cache or timestamps as content identity. A full bare clone gives one immutable
422
429
  // Git commit/tree; every observed raw_url's blob identity must match its HEAD file bytes.
423
- const snapshot = await fetchGitSnapshot({ owner: observation.owner, stub, signal });
430
+ const snapshot = await retryTransientAsync(() => fetchGitSnapshot({ owner: observation.owner, stub, signal }),
431
+ { isTransient: transientGistSnapshot, baseDelayMs: snapshotRetryDelayMs });
424
432
  const files = [];
425
433
  for (const source of snapshot.files) {
426
434
  const filename = source.filename;
@@ -71,6 +71,7 @@ import {
71
71
  import { bindCoverageToReceipt, writeCoverageAssets } from './corpus-coverage-sidecar.mjs';
72
72
  import { describeDegraded } from './corpus-store-failure.mjs';
73
73
  import { assertNoNewerCorpusGeneration } from './code-release-corpus.mjs';
74
+ import { retryingGh } from './transient-gh.mjs';
74
75
 
75
76
  const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
76
77
  const PUBLISH = process.argv.includes('--publish');
@@ -141,11 +142,17 @@ const compareCodeTags = (left, right) => {
141
142
  return Math.sign(a[0] - b[0] || a[1] - b[1] || a[2] - b[2]);
142
143
  };
143
144
 
144
- /** The one gh seam both corpus entry points use (RUVNET_GH_COMMAND / RUVNET_GH_SCRIPT for tests). */
145
+ /**
146
+ * The one gh seam both corpus entry points use (RUVNET_GH_COMMAND / RUVNET_GH_SCRIPT for tests). Reads and the
147
+ * idempotent `release edit` retry a transient failure, bounded (scripts/transient-gh.mjs): on 2026-09-29 the
148
+ * night was red on "cannot confirm the promoted corpus release" -- one `release view` that failed right after
149
+ * the write had succeeded. `release create` runs exactly once; the absence proof above it is what makes a
150
+ * repeat safe to refuse.
151
+ */
145
152
  function corpusGh(env, run) {
146
- const ghCommand = env.RUVNET_GH_COMMAND || 'gh';
147
- const ghPrefix = env.RUVNET_GH_SCRIPT ? [env.RUVNET_GH_SCRIPT] : [];
148
- const gh = (args) => run(ghCommand, [...ghPrefix, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
153
+ const gh = retryingGh({ run, command: env.RUVNET_GH_COMMAND || 'gh', prefix: env.RUVNET_GH_SCRIPT ? [env.RUVNET_GH_SCRIPT] : [],
154
+ options: { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] },
155
+ onRetry: ({ attempt, args, delayMs }) => console.error(`[corpus-seed] gh ${args.slice(0, 2).join(' ')} failed transiently (attempt ${attempt}); retrying in ${delayMs / 1000}s`) });
149
156
  const ghJson = (args, label) => {
150
157
  const result = gh(args);
151
158
  if (result.error || result.status !== 0) {
@@ -9,6 +9,7 @@ import { readRvfGenerations, sha256File } from './rvf-generation.mjs';
9
9
  import { canonicalJson, coverageGenerationFor, digest, validateGistAggregateReceipt } from './coverage-integrity.mjs';
10
10
  import { repositoryNames } from '../kb/card-lane.mjs';
11
11
  import { rootNeverMaterialized, storeRoot } from '../kb/store-root.mjs';
12
+ import { isTransientText, retryTransient, sleepSync } from './transient-gh.mjs';
12
13
 
13
14
  export { canonicalJson, digest } from './coverage-integrity.mjs';
14
15
 
@@ -52,6 +53,11 @@ function runGh(args) {
52
53
  return result.stdout;
53
54
  }
54
55
 
56
+ // Every observation read below goes through this: a transient failure (rate limit, 5xx, a reset) is retried,
57
+ // bounded, before it can fail the night (scripts/transient-gh.mjs). Applied around the INJECTED `gh`, not
58
+ // inside runGh, so the fault-injection suite exercises the same retry the nightly runs.
59
+ const retryingRead = (gh, sleep) => (args) => retryTransient(() => gh(args), { sleep });
60
+
55
61
  // A fork's disposition depends on how many commits it carries that its upstream does not. The
56
62
  // compare is pinned to the two heads THIS observation recorded (never to branch names, which move),
57
63
  // and is run against the upstream repository in GitHub's documented cross-fork form
@@ -80,7 +86,8 @@ function observeForkDelta(owner, node, gh) {
80
86
  };
81
87
  }
82
88
 
83
- export function observeRepositories(owner, { gh = runGh } = {}) {
89
+ export function observeRepositories(owner, { gh: rawGh = runGh, sleep = sleepSync } = {}) {
90
+ const gh = retryingRead(rawGh, sleep);
84
91
  const rows = [];
85
92
  const pages = [];
86
93
  let cursor = '';
@@ -103,10 +110,11 @@ export function observeRepositories(owner, { gh = runGh } = {}) {
103
110
  return { rows: withDeltas, expected, pages };
104
111
  }
105
112
 
106
- // The row an external source reports once its upstream is gone: exactly what was last observed, so the
107
- // stored content classifies CURRENT, nothing is re-ingested, and the row says plainly it is no longer live.
108
- function goneExternalRow(source) {
109
- const gone = source.ifGone;
113
+ // The row an external source reports when its upstream cannot be read: exactly what was last observed (or last
114
+ // ingested), so the stored content classifies CURRENT, nothing is re-ingested, and the row says plainly why.
115
+ // `gone` is the policy's ifGone entry (404/410); for a source unreadable for a moment it is the seed ledger's
116
+ // own commit, with `reason` naming the failure.
117
+ function unavailableExternalRow(source, gone, reason) {
110
118
  if (!/^[0-9a-f]{40}$/i.test(String(gone.sourceCommit || ''))) {
111
119
  throw new Error(`external source ${source.repository}: ifGone.sourceCommit must be the 40-hex commit last ingested`);
112
120
  }
@@ -114,12 +122,27 @@ function goneExternalRow(source) {
114
122
  name: source.repository.split('/')[1], fullName: source.repository, storeName: source.store,
115
123
  url: `https://github.com/${source.repository}`, description: gone.description ?? null, homepageUrl: null,
116
124
  isFork: false, upstreamIsFork: false, isArchived: false, isDisabled: false,
117
- updatedAt: gone.updatedAt ?? null, pushedAt: gone.pushedAt ?? null, upstreamUnavailable: true,
125
+ updatedAt: gone.updatedAt ?? null, pushedAt: gone.pushedAt ?? null, upstreamUnavailable: true, upstreamUnavailableReason: reason,
118
126
  defaultBranchRef: { name: gone.defaultBranch || 'HEAD', target: { oid: gone.sourceCommit, committedDate: gone.committedAt ?? null } },
119
127
  };
120
128
  }
121
129
 
122
- export function observeExternalRepositories(sources, { gh = runGh } = {}) {
130
+ /** The one reason a transient carry records; scripts/corpus-reconcile.mjs refuses a second consecutive one by it. */
131
+ export const TRANSIENT_UNREADABLE_REASON = 'was unreadable this observation (transient failure after the bounded retry); it is read again next night';
132
+ /** The carry record a transiently unreadable source ships: STALE, not CURRENT, and counted toward the degraded bound. */
133
+ export const TRANSIENT_CARRY_REASON = 'transient: upstream unreadable';
134
+
135
+ /**
136
+ * `carriedCommits`: { <store>: { sourceCommit, committedAt } } from the seed's own generation ledger, already stripped
137
+ * of any store the seed's coverage shows was ALREADY carried this way (corpus-reconcile.mjs transientCarryCommits),
138
+ * so the same source is never carried two nights running. A source whose read still fails TRANSIENTLY after the
139
+ * bounded retry (rate limit, 5xx, a reset; measured 2026-09-26) is pinned at that commit for ONE night: its row
140
+ * classifies STALE with a `carry` record (nobody read upstream tonight, so it is not called current), it counts
141
+ * toward max(3, 5% of eligible) with every other carried store, and the next night reads it again. Without a prior
142
+ * commit there is nothing honest to carry, and the observation fails.
143
+ */
144
+ export function observeExternalRepositories(sources, { gh: rawGh = runGh, sleep = sleepSync, carriedCommits = null } = {}) {
145
+ const gh = retryingRead(rawGh, sleep);
123
146
  const rows = [];
124
147
  const pages = [];
125
148
  for (const source of sources) {
@@ -134,13 +157,28 @@ export function observeExternalRepositories(sources, { gh = runGh } = {}) {
134
157
  ? JSON.parse(gh(['api', `repos/${source.repository}/commits/${repo.default_branch}`]))
135
158
  : null;
136
159
  } catch (error) {
137
- // An upstream that GitHub reports gone (404/410) must not stop every night for every customer.
138
- // Only a source whose policy entry records `ifGone` (the commit it was last ingested at) may be
139
- // carried; anything else -- 403, rate limit, a mistyped repository -- still fails the observation.
140
- if (!source.ifGone || !/\(HTTP (404|410)\)/.test(String(error.message))) throw error;
141
- process.stderr.write(`[source-coverage] external source ${source.repository} is gone upstream; carrying the last ingested content (${source.ifGone.sourceCommit})\n`);
142
- rows.push(goneExternalRow(source));
143
- pages.push({ repository: source.repository, responseDigest: digest({ gone: source.repository, ifGone: source.ifGone }), count: 1, terminal: true });
160
+ // An upstream that GitHub reports gone (404/410) must not stop every night for every customer: a source
161
+ // whose policy entry records `ifGone` (the commit it was last ingested at) is carried there. A source that
162
+ // is merely unreadable tonight (still transient after the retry) is carried at the seed's own commit when
163
+ // it has one. Anything else -- a bare 403, a mistyped repository, no prior bytes -- still fails the observation.
164
+ const message = String(error.message);
165
+ const gone = source.ifGone && /\(HTTP (404|410)\)/.test(message);
166
+ const prior = carriedCommits?.[String(source.store).toLowerCase()];
167
+ const carried = String(prior?.sourceCommit ?? prior ?? '').toLowerCase();
168
+ const unreadable = !gone && isTransientText(message) && /^[0-9a-f]{40}$/.test(carried);
169
+ if (!gone && !unreadable) throw error;
170
+ // FIXED shapes, never the error text or a timestamp: the observation digest feeds the no-change decision,
171
+ // so a night that carried one unreadable source and saw nothing else move must still read as unchanged.
172
+ const kind = gone ? 'gone' : 'transient';
173
+ const reason = gone ? 'is no longer reachable (HTTP 404/410)' : TRANSIENT_UNREADABLE_REASON;
174
+ // committedAt comes from the seed's own coverage row (corpus-reconcile.mjs carriedCommittedAt), so the customer's
175
+ // "oldest carried" line dates the pinned commit instead of showing an unknown date.
176
+ const at = gone ? source.ifGone : { sourceCommit: carried, committedAt: prior?.committedAt ?? null };
177
+ process.stderr.write(`[source-coverage] external source ${source.repository} ${reason}; carrying the last ingested content (${at.sourceCommit})`
178
+ + `${gone ? '' : ` -- ${message.replace(/\s+/g, ' ').trim().slice(0, 200)}`}\n`);
179
+ rows.push({ ...unavailableExternalRow(source, at, reason), upstreamUnavailableKind: kind });
180
+ pages.push({ repository: source.repository, responseDigest: digest(gone ? { gone: source.repository, ifGone: source.ifGone }
181
+ : { unavailable: source.repository, kind, at: carried }), count: 1, terminal: true });
144
182
  continue;
145
183
  }
146
184
  const row = {
@@ -164,9 +202,11 @@ export function observeSourceUniverse({
164
202
  externalSources = [],
165
203
  gh = runGh,
166
204
  observedAt = new Date().toISOString(),
205
+ sleep = sleepSync,
206
+ carriedCommits = null,
167
207
  } = {}) {
168
- const primary = observeRepositories(owner, { gh });
169
- const external = observeExternalRepositories(externalSources, { gh });
208
+ const primary = observeRepositories(owner, { gh, sleep });
209
+ const external = observeExternalRepositories(externalSources, { gh, sleep, carriedCommits });
170
210
  const repositoryRows = canonicalRepositoryRows([...primary.rows, ...external.rows]);
171
211
  const repositories = sourceSetObservation(repositoryRows, primary.expected + external.expected);
172
212
  const repositoryKeys = new Set();
@@ -179,7 +219,7 @@ export function observeSourceUniverse({
179
219
  repositoryKeys.add(repositoryKey);
180
220
  storeKeys.add(storeKey);
181
221
  }
182
- const observedGists = observeGists(owner, { gh });
222
+ const observedGists = observeGists(owner, { gh, sleep });
183
223
  const gists = sourceSetObservation(canonicalGistRows(observedGists.rows), observedGists.expected);
184
224
  return canonicalSourceObservation({
185
225
  schemaVersion: 1,
@@ -222,6 +262,8 @@ export function canonicalRepositoryRows(rows = []) {
222
262
  isArchived: repo?.isArchived === true,
223
263
  isDisabled: repo?.isDisabled === true,
224
264
  ...(repo?.upstreamUnavailable === true ? { upstreamUnavailable: true } : {}),
265
+ ...(repo?.upstreamUnavailable === true && repo?.upstreamUnavailableReason ? { upstreamUnavailableReason: String(repo.upstreamUnavailableReason) } : {}),
266
+ ...(repo?.upstreamUnavailable === true && repo?.upstreamUnavailableKind ? { upstreamUnavailableKind: String(repo.upstreamUnavailableKind) } : {}),
225
267
  diskUsage: repo?.diskUsage,
226
268
  updatedAt: repo?.updatedAt ?? null,
227
269
  pushedAt: repo?.pushedAt ?? null,
@@ -325,7 +367,11 @@ function listGistsUnauthenticated(owner, curl) {
325
367
  return pages;
326
368
  }
327
369
 
328
- export function observeGists(owner, { gh = runGh, curl = runCurl } = {}) {
370
+ export function observeGists(owner, { gh: rawGh = runGh, curl: rawCurl = runCurl, sleep = sleepSync } = {}) {
371
+ // A rate limit on either channel (measured 2026-09-26: the gist list) is retried, bounded, before the
372
+ // integration-token 403 is allowed to fall back to the anonymous list, which is itself rate-limited.
373
+ const gh = retryingRead(rawGh, sleep);
374
+ const curl = retryingRead(rawCurl, sleep);
329
375
  let pages;
330
376
  try {
331
377
  const raw = gh(['api', `users/${owner}/gists?per_page=100`, '--paginate', '--slurp']);
@@ -414,8 +460,19 @@ export function classifyRepository(repo, evidence, exclusion = null, outcome = n
414
460
  status = 'FAILED';
415
461
  reasons.push(outcome.integrity);
416
462
  }
417
- if (repo.upstreamUnavailable === true) reasons.push(`upstream ${repo.fullName} is no longer reachable (HTTP 404/410); serving the content last ingested at ${upstreamSha}`);
463
+ if (repo.upstreamUnavailable === true) {
464
+ reasons.push(`upstream ${repo.fullName} ${repo.upstreamUnavailableReason || 'is no longer reachable (HTTP 404/410)'}; serving the content last ingested at ${upstreamSha}`);
465
+ }
418
466
  const record = {};
467
+ // A source nobody could read tonight is never CURRENT: its bytes are pinned at the last commit the seed carried,
468
+ // and the row ships a carry record (missedUpstream null -- the upstream head is unknown, not merely different)
469
+ // that eligibleRepositoryStanding accepts only together with upstreamUnavailable. A `gone` source stays CURRENT:
470
+ // its upstream was READ and is final.
471
+ if (disposed && repo.upstreamUnavailableKind === 'transient' && status === 'CURRENT') {
472
+ status = 'STALE';
473
+ record.carry = { reason: TRANSIENT_CARRY_REASON, carriedSourceCommit: upstreamSha, missedUpstream: null,
474
+ attempts: 1, carriedCommittedAt: repo.defaultBranchRef?.target?.committedDate || null };
475
+ }
419
476
  if (disposed && outcome?.carry && status === 'STALE') record.carry = { ...outcome.carry };
420
477
  if (disposed && outcome?.failure && status === 'MISSING') record.failure = { ...outcome.failure };
421
478
  return {
@@ -428,6 +485,7 @@ export function classifyRepository(repo, evidence, exclusion = null, outcome = n
428
485
  disposition,
429
486
  archived: repo.isArchived === true,
430
487
  ...(repo.upstreamUnavailable === true ? { upstreamUnavailable: true } : {}),
488
+ ...(repo.upstreamUnavailable === true && repo.upstreamUnavailableKind ? { upstreamUnavailableKind: repo.upstreamUnavailableKind } : {}),
431
489
  ...(repo.forkDelta ? { forkDelta: { ...repo.forkDelta } } : {}),
432
490
  upstream: { sha: upstreamSha, committedAt: repo.defaultBranchRef?.target?.committedDate || null,
433
491
  pushedAt: repo.pushedAt, updatedAt: repo.updatedAt },
@@ -0,0 +1,150 @@
1
+ // DISTINCT-FROM: kb/download-retry.mjs — that retries the customer updater's HTTPS stream by error code; this
2
+ // classifies and retries the pipeline's `gh`/`git` subprocess reads by what they printed.
3
+ //
4
+ // scripts/transient-gh.mjs — ONE rule for what a transient GitHub/API/git failure looks like, and ONE bounded
5
+ // retry for it, shared by every read in the corpus nightly (ADR-0091, fault classes in CONTRIBUTING.md).
6
+ //
7
+ // THE FAILURE THIS EXISTS TO END (measured on 38 corpus nights, 2026-09-21 .. 2026-10-18). Twenty-four nights
8
+ // failed; the causes were a gist API rate limit, a publisher that could not re-read the release it had just
9
+ // created, a canary refused one anonymous read, a git credential prompt on a clone -- each a single blip at a
10
+ // single `gh`/`git`/`fetch` call, each unretried, each the whole night for every customer. Before this module
11
+ // every call site either had no retry or its own private idea of "transient" (the canary's status set, the
12
+ // gist transport's regexes, the reconciler's exit-status classes). The rule is now written once:
13
+ //
14
+ // TRANSIENT a rate limit (primary or secondary), HTTP 429/408/5xx, a connection reset/refused/timed out, a
15
+ // DNS hiccup, a TLS handshake failure, a hung-up socket, git's credential prompt on an anonymous
16
+ // fetch ("could not read Username"), a partial transfer ("early EOF", "RPC failed").
17
+ // NOT HTTP 404/410 (gone: the ifGone carry in source-coverage.mjs owns it), a bare HTTP 403 from an
18
+ // authenticated gh (a permission answer, e.g. "Resource not accessible by integration"), "release
19
+ // not found" (the publisher's absence proof), a signature, digest or schema refusal, a missing
20
+ // binary (ENOENT). Those are answers, not weather; retrying them only delays the honest failure.
21
+ //
22
+ // Anonymous `fetch` reads from a shared runner address are the one place a bare 403 IS transient (GitHub's
23
+ // per-IP throttle answers 403): that status set, TRANSIENT_HTTP, lives here too so the canary and any other
24
+ // anonymous reader share it rather than restating it.
25
+ //
26
+ // WRITES ARE NEVER RETRIED BLINDLY. `gh release create` uploads assets; a retry after a half-finished create
27
+ // collides with the release it left behind. `retryingGh` retries reads and idempotent edits only; a create or
28
+ // any `gh api` POST/PUT/PATCH/DELETE runs exactly once and reports what happened.
29
+
30
+ /** Anonymous-fetch statuses a shared runner address sees for a moment (GitHub throttles anonymous reads with 403). */
31
+ export const TRANSIENT_HTTP = new Set([403, 408, 429, 500, 502, 503, 504]);
32
+
33
+ export const DEFAULT_ATTEMPTS = 3;
34
+ export const DEFAULT_BASE_DELAY_MS = 15_000;
35
+ /** The wait before retry `n` is `base * n`. RUVNET_TRANSIENT_RETRY_DELAY_MS is the test seam (the suite sets 0 so a
36
+ * fake gh that answers "gateway timeout" fails in milliseconds, not after the 45 s a real night would wait). */
37
+ export function defaultBaseDelayMs(fallback = DEFAULT_BASE_DELAY_MS) {
38
+ const seam = Number(process.env.RUVNET_TRANSIENT_RETRY_DELAY_MS);
39
+ return Number.isFinite(seam) && seam >= 0 && process.env.RUVNET_TRANSIENT_RETRY_DELAY_MS !== undefined ? seam : fallback;
40
+ }
41
+
42
+ const TRANSIENT_TEXT = /\brate limit|secondary rate|abuse detection|HTTP (408|429|5\d\d)\b|\b(500|502|503|504) (Internal Server Error|Bad Gateway|Service Unavailable|Gateway Time-?out)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|EPIPE|connection reset|connection refused|timed out|timeout|TLS handshake|socket hang up|could not resolve host|no such host|dial tcp|i\/o timeout|unexpected EOF|early EOF|RPC failed|remote end hung up|could not read Username|unable to access|temporarily unavailable|temporary failure/i;
43
+
44
+ /** What a failed subprocess said -- stderr, then stdout, then the spawn error -- as one string for classification. */
45
+ export function failureText(result) {
46
+ return [result?.stderr, result?.stdout, result?.error?.message].map((part) => String(part ?? '')).join('\n');
47
+ }
48
+
49
+ /** The one classifier: true when the text describes weather, not an answer. A missing binary is never weather. */
50
+ export function isTransientText(text) {
51
+ const message = String(text ?? '');
52
+ if (/\bENOENT\b/.test(message)) return false;
53
+ return TRANSIENT_TEXT.test(message);
54
+ }
55
+
56
+ /** A spawnSync-shaped result that failed transiently. */
57
+ export function isTransientResult(result) {
58
+ if (!result) return false;
59
+ if (!result.error && result.status === 0) return false;
60
+ if (result.error?.code === 'ENOENT') return false;
61
+ return isTransientText(failureText(result));
62
+ }
63
+
64
+ /** Blocks the thread for `ms`; the pipeline's gh/git callers are synchronous spawnSync loops. Injectable in tests. */
65
+ export function sleepSync(ms) {
66
+ if (ms > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
67
+ }
68
+
69
+ const delayFor = (attempt, baseDelayMs) => baseDelayMs * attempt;
70
+
71
+ /**
72
+ * Run `fn` (which THROWS on failure) up to `attempts` times while the failure is transient, waiting
73
+ * baseDelayMs * attempt between tries. The last error is rethrown unchanged, so callers keep their own messages.
74
+ */
75
+ export function retryTransient(fn, { attempts = DEFAULT_ATTEMPTS, baseDelayMs = defaultBaseDelayMs(), sleep = sleepSync,
76
+ isTransient = (error) => isTransientText(error?.message), onRetry = () => {} } = {}) {
77
+ for (let attempt = 1; ; attempt += 1) {
78
+ try {
79
+ return fn(attempt);
80
+ } catch (error) {
81
+ if (attempt >= attempts || !isTransient(error)) throw error;
82
+ onRetry({ attempt, error, delayMs: delayFor(attempt, baseDelayMs) });
83
+ sleep(delayFor(attempt, baseDelayMs));
84
+ }
85
+ }
86
+ }
87
+
88
+ /** The async twin of retryTransient, for callers whose work (or sleep) is a promise. */
89
+ export async function retryTransientAsync(fn, { attempts = DEFAULT_ATTEMPTS, baseDelayMs = defaultBaseDelayMs(),
90
+ sleep = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); }),
91
+ isTransient = (error) => isTransientText(error?.message), onRetry = () => {} } = {}) {
92
+ for (let attempt = 1; ; attempt += 1) {
93
+ try {
94
+ return await fn(attempt);
95
+ } catch (error) {
96
+ if (attempt >= attempts || !isTransient(error)) throw error;
97
+ onRetry({ attempt, error, delayMs: delayFor(attempt, baseDelayMs) });
98
+ await sleep(delayFor(attempt, baseDelayMs));
99
+ }
100
+ }
101
+ }
102
+
103
+ /**
104
+ * Run `fn` (which RETURNS a spawnSync-shaped result, never throws) up to `attempts` times while the result is a
105
+ * transient failure. Returns the last result, so callers that read `status`/`stderr` keep doing exactly that.
106
+ */
107
+ export function retryTransientResult(fn, { attempts = DEFAULT_ATTEMPTS, baseDelayMs = defaultBaseDelayMs(),
108
+ sleep = sleepSync, onRetry = () => {} } = {}) {
109
+ let result;
110
+ for (let attempt = 1; ; attempt += 1) {
111
+ result = fn(attempt) || {};
112
+ if (attempt >= attempts || !isTransientResult(result)) return result;
113
+ onRetry({ attempt, result, delayMs: delayFor(attempt, baseDelayMs) });
114
+ sleep(delayFor(attempt, baseDelayMs));
115
+ }
116
+ }
117
+
118
+ const GH_WRITE_VERBS = new Set(['create', 'delete', 'upload', 'delete-asset']);
119
+ const REST_WRITE_FLAGS = new Set(['-f', '-F', '--input', '--field', '--raw-field']);
120
+ const METHOD_FLAGS = new Set(['-X', '--method']);
121
+
122
+ /**
123
+ * Is this `gh` argument vector a write that must run exactly once? `release edit` is an idempotent PATCH and is
124
+ * NOT a write here: re-sending the same flags after a timeout converges on the same state. `gh api` is a write
125
+ * when it names a non-GET method or carries a field flag (gh then sends POST); `gh api graphql` always POSTs.
126
+ */
127
+ export function isGhWrite(args) {
128
+ const argv = (args || []).map(String);
129
+ if (argv[0] === 'api') {
130
+ const methodAt = argv.findIndex((token) => METHOD_FLAGS.has(token));
131
+ if (methodAt >= 0 && String(argv[methodAt + 1] || '').toUpperCase() !== 'GET') return true;
132
+ if (argv[1] === 'graphql') return true;
133
+ return argv.some((token) => REST_WRITE_FLAGS.has(token));
134
+ }
135
+ return argv[0] === 'release' && GH_WRITE_VERBS.has(argv[1]);
136
+ }
137
+
138
+ /**
139
+ * A `gh` runner over a spawnSync-shaped `run` that retries transient failures of READS and idempotent edits,
140
+ * bounded, and runs a write exactly once. `prefix` is the RUVNET_GH_SCRIPT test seam (scripts/release.mjs).
141
+ */
142
+ export function retryingGh({ run, command = 'gh', prefix = [], options = {}, sleep = sleepSync, attempts = DEFAULT_ATTEMPTS,
143
+ baseDelayMs = defaultBaseDelayMs(), onRetry = () => {} } = {}) {
144
+ if (typeof run !== 'function') throw new Error('retryingGh needs a spawnSync-shaped run function');
145
+ return (args) => {
146
+ const once = () => run(command, [...prefix, ...args], options) || {};
147
+ if (isGhWrite(args)) return once();
148
+ return retryTransientResult(once, { attempts, baseDelayMs, sleep, onRetry: (event) => onRetry({ ...event, args }) });
149
+ };
150
+ }