ruvnet-brain 4.7.15 → 4.7.17

package/scripts/corpus-reconcile.mjs

@@ -25,6 +25,7 @@ import { storeRoot } from '../kb/store-root.mjs';
 import { captureGistSources } from './gist-receipts.mjs';
 import { projectSourceStore, RUNTIME_LEDGER_KIND } from './rvf-generation.mjs';
 import { compareKnowledgeInputs, fromCoverage as knowledgeFromCoverage, fromSeed as knowledgeFromSeed } from './knowledge-input-digest.mjs';
+import { defaultBaseDelayMs } from './transient-gh.mjs';
 
 export { rebuildCorpusAggregates };
 
@@ -352,8 +353,10 @@ export async function acquireSealedGeneration({ maxAttempts = 3, assetsDir = nul
     const settled = await build(observation, storeOutcomes);
     const remaining = planReconciliation({ coverage: settled, ledger: currentLedger(), assetsDir })
       .filter((item) => !recorded(item.store));
+    // A repository row the shipped validator accepts WITH a carry -- a transiently unreadable source pinned at its
+    // seed commit (source-coverage.mjs) -- is resolved for this generation, exactly like a recorded store outcome.
     const unresolved = settled.rows.filter((row) => row.disposition === 'eligible' && row.status !== 'CURRENT'
-      && !(row.kind === 'repository' && recorded(row.artifact?.store)));
+      && !(row.kind === 'repository' && (recorded(row.artifact?.store) || (row.carry && eligibleRepositoryStanding(row) === 'shipped'))));
     attempts.push({ attempt, plan, ...reconciliation, ...pruning, ...aggregates,
       remainingArtifacts: remaining.length, unresolvedSources: unresolved.length });
     if (!remaining.length && !unresolved.length) {
@@ -635,11 +638,13 @@ export function dispositionForFailedStore({ assetsDir, ledger, item, attempts, r
 
 // Only a TRANSIENT failure is retried, exactly once, in a fresh directory: `<store>-retry1`. The first
 // attempt's directory is never reused -- a clone or a half-written worker output from the failed
-// attempt must not be mistaken for the retry's own (the pre-D5 path collided).
+// attempt must not be mistaken for the retry's own (the pre-D5 path collided). The retry waits
+// TRANSIENT_RETRY_DELAY_MS first: an immediate re-clone into the same reset connection learns nothing.
 export function workerRootFor(workspace, store, retry) {
   return path.join(workspace, 'workers', retry === 0 ? store : `${store}-retry${retry}`);
 }
 export const MAX_TRANSIENT_RETRIES = 1;
+export const TRANSIENT_RETRY_DELAY_MS = 30_000;
 
 export async function executeReconciliation({
   plan,
@@ -651,6 +656,8 @@ export async function executeReconciliation({
   signal,
   priorCoverage = null,
   log = (line) => console.log(line),
+  retryDelayMs = defaultBaseDelayMs(TRANSIENT_RETRY_DELAY_MS),
+  sleep = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); }),
 }) {
   if (!Array.isArray(plan) || !Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 10) {
     fail('reconciliation plan or worker concurrency is invalid');
@@ -750,8 +757,10 @@ export async function executeReconciliation({
         lastError = error;
         const retrying = isRetryable(error) && retry < MAX_TRANSIENT_RETRIES;
         log(`[corpus-reconcile] ${item.store}: attempt ${attempts} failed -- ${error.message}`
-          + (retrying ? '; retrying once in a fresh worker directory' : '; not retried'));
+          + (retrying ? `; retrying once in a fresh worker directory after ${retryDelayMs / 1000}s` : '; not retried'));
         if (!retrying) break;
+        await sleep(retryDelayMs);
+        if (controller.signal.aborted) throw abortError(controller.signal);
       }
     }
     return { ok: false, attempts, error: lastError };
@@ -871,7 +880,33 @@ async function observeSourceOnly({ owner, assetsDir }) {
   const externalFile = path.join(assets, 'external-sources.json');
   const policy = fs.existsSync(externalFile) ? readJson(externalFile, 'external source policy') : { sources: [] };
   if (!Array.isArray(policy.sources)) fail('external source policy has no sources array');
-  return observeSourceUniverse({ owner, externalSources: policy.sources });
+  const ledgerFile = path.join(assets, 'RVF-GENERATIONS.json');
+  const ledger = fs.existsSync(ledgerFile) ? readJson(ledgerFile, 'RVF generation ledger') : { stores: {} };
+  return observeSourceUniverse({ owner, externalSources: policy.sources,
+    carriedCommits: transientCarryCommits({ ledger, priorCoverage: readPriorCoverage(assets) }) });
+}
+
+/**
+ * The seed's own commits, per store: what an external source unreadable tonight is pinned at (one night, STALE
+ * with a carry) instead of failing the whole observation. Read from the seed ledger, never invented -- and a store
+ * the SEED's sealed coverage already shows carried this way is left out, so the second consecutive night fails
+ * for that source exactly as before rather than carrying it forever.
+ */
+export function transientCarryCommits({ ledger, priorCoverage = null }) {
+  // No sealed coverage to read history from (the bootstrap lineage, or an invalid sidecar) = no carry at all:
+  // the allowance exists only where the previous night can prove it was not already spent.
+  if (!Array.isArray(priorCoverage?.rows)) return {};
+  // ANY carried seed row spends it -- a failed refresh last night and an unreadable upstream tonight must not
+  // alternate forever either.
+  const alreadyCarried = new Set(priorCoverage.rows
+    .filter((row) => row?.kind === 'repository' && row.carry)
+    .map((row) => String(row.artifact?.store || '').toLowerCase()));
+  return Object.fromEntries(Object.entries(ledger?.stores || {})
+    .filter(([store, row]) => HEX40.test(String(row?.sourceCommit || '').toLowerCase()) && !alreadyCarried.has(store.toLowerCase()))
+    .map(([store, row]) => {
+      const sourceCommit = String(row.sourceCommit).toLowerCase();
+      return [store.toLowerCase(), { sourceCommit, committedAt: carriedCommittedAt({ priorCoverage, store, sourceCommit }) }];
+    }));
 }
 
 export async function acquireCorpusGeneration({ owner = 'ruvnet', assetsDir, workspaceDir,

package/scripts/corpus-watchdog.mjs

@@ -22,6 +22,10 @@
 //   WARNING  a superseded or degraded night, a stand-down tonight, an unknown outcome, a long run
 //   GREEN    none of the above
 //
+// THE FIRST LINE IS THE NIGHT'S VERDICT (2026-10-20, summarizeNight): what the night did -- published,
+// no-change, degraded with the stores named, failed with the stage named, superseded, canary-rejected, still
+// running, stood down -- in Eastern time, result first, so the owner never learns of a break from a page.
+//
 // Exit code is the contract (same as scripts/github-health-watch.mjs): 0 = GREEN or WARNING,
 // 1 = RED or the evidence could not be gathered. corpus-watchdog.yml runs on a genuine `schedule:`,
 // so its red conclusion fires ntfy-alerts.yml's workflow_run listener — no secret needed here.
@@ -43,6 +47,7 @@ import { CODE_TAG_PATTERN, isCorpusReleaseTag } from './release-channel-kind.mjs
 import { AGGREGATE_ASSET, SIGNING_PUBLIC_KEY_FILE } from './approved-runtime.mjs';
 import { COVERAGE_ASSET, COVERAGE_RECEIPT_ASSET, verifyCoverageSidecar } from './corpus-coverage-sidecar.mjs';
 import { parseCorpusGeneration } from './corpus-promotion.mjs';
+import { retryTransientResult } from './transient-gh.mjs';
 
 const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 export const RED = 'RED';
@@ -255,7 +260,9 @@ export function judgeCorpusHealth(input, now) {
 // ── Gathering (read-only gh). Everything above is pure; everything below talks to GitHub. ──────────
 
 function defaultGh(args) {
-  const result = spawnSync(process.env.RUVNET_GH_COMMAND || 'gh', args, { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 5 * 60_000 });
+  // A transient gh failure is retried, bounded (scripts/transient-gh.mjs); the judge must not page RED on weather.
+  const result = retryTransientResult(() => spawnSync(process.env.RUVNET_GH_COMMAND || 'gh', args,
+    { encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 5 * 60_000 }));
   if (result.error || result.status !== 0) {
     throw new Error(`gh ${args.slice(0, 3).join(' ')} failed: ${String(result.error?.message || result.stderr || `exit ${result.status}`).trim().slice(0, 300)}`);
   }
@@ -320,17 +327,25 @@ function corpusRunEvidence({ gh, repo, run, scratch }) {
   }
 }
 
+/**
+ * The protected-release corpus runs created since `since` (ms), each with its outcome evidence. Shared with the
+ * dispatcher's retry slot (scripts/corpus-dispatch-decision.mjs --retry-slot), so both read one definition of
+ * "tonight's runs" and one classifier.
+ */
+export function gatherCorpusRuns({ repo, since, gh = defaultGh, scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'corpus-watchdog-')) }) {
+  return JSON.parse(gh(['run', 'list', '--repo', repo, '--workflow', 'protected-release.yml', '--limit', '60',
+    '--json', 'databaseId,attempt,displayTitle,status,conclusion,createdAt,updatedAt']))
+    .filter((run) => CORPUS_RUN_TITLE.test(String(run.displayTitle || '')) && (ms(run.createdAt) ?? 0) >= since)
+    .map((run) => ({ id: run.databaseId, title: run.displayTitle, status: run.status, conclusion: run.conclusion,
+      createdAt: run.createdAt, updatedAt: run.updatedAt, ...corpusRunEvidence({ gh, repo, run, scratch }) }));
+}
+
 export async function gatherCorpusHealthInput({ repo, now, gh = defaultGh, root = ROOT, scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'corpus-watchdog-')) }) {
   const at = now.getTime();
-  const recent = (iso) => (ms(iso) ?? 0) >= at - 3 * 24 * HOUR;
   const dispatcherRuns = JSON.parse(gh(['run', 'list', '--repo', repo, '--workflow', 'corpus-nightly-dispatch.yml', '--limit', '30',
     '--json', 'databaseId,attempt,event,status,conclusion,createdAt,updatedAt']))
     .map((run) => ({ id: run.databaseId, attempt: run.attempt, event: run.event, status: run.status, conclusion: run.conclusion, createdAt: run.createdAt }));
-  const corpusRuns = JSON.parse(gh(['run', 'list', '--repo', repo, '--workflow', 'protected-release.yml', '--limit', '60',
-    '--json', 'databaseId,attempt,displayTitle,status,conclusion,createdAt,updatedAt']))
-    .filter((run) => CORPUS_RUN_TITLE.test(String(run.displayTitle || '')) && recent(run.createdAt))
-    .map((run) => ({ id: run.databaseId, title: run.displayTitle, status: run.status, conclusion: run.conclusion,
-      createdAt: run.createdAt, updatedAt: run.updatedAt, ...corpusRunEvidence({ gh, repo, run, scratch }) }));
+  const corpusRuns = gatherCorpusRuns({ repo, since: at - 3 * 24 * HOUR, gh, scratch });
   const releases = JSON.parse(gh(['release', 'list', '--repo', repo, '--limit', '100', '--json', 'tagName,publishedAt,isDraft,isPrerelease']));
   // PROMOTED only: a staged candidate (and a bootstrap seed) is a prerelease no customer receives.
   const corpusReleases = releases.filter((row) => !row.isDraft && !row.isPrerelease && isCorpusReleaseTag(row.tagName))
@@ -364,8 +379,98 @@ export async function gatherCorpusHealthInput({ repo, now, gh = defaultGh, root
   return { corpusReleases, corpusRuns, dispatcherRuns, codeRelease, deferral, latestRelease };
 }
 
+/** A time the owner reads: local Eastern, 12-hour, named zone -- never UTC or "Z". */
+export function formatEastern(when) {
+  const t = when instanceof Date ? when.getTime() : ms(when);
+  if (t === null || !Number.isFinite(t)) return 'an unknown time';
+  const parts = Object.fromEntries(new Intl.DateTimeFormat('en-US', { timeZone: 'America/New_York', month: 'short', day: 'numeric',
+    hour: 'numeric', minute: '2-digit', hour12: true }).formatToParts(new Date(t)).map((part) => [part.type, part.value]));
+  return `${parts.hour}:${parts.minute} ${parts.dayPeriod.toLowerCase().replace(/^(a|p)m$/, '$1.m.')} ET ${parts.month} ${parts.day}`;
+}
+
+/** The job the night died in, named as the owner knows the stage, from the terminal outcome record or the job list. */
+const STAGE_NAMES = { identity: 'authorize (corpus-identity)', prepare: 'prepare/seal (seal-exact-corpus-candidate)',
+  authorize: 'authenticate (corpus-authorize)', publish: 'publisher (corpus-publish)', canary: 'customer canary', promote: 'promote' };
+const JOB_NAMES = { 'authorize-corpus-generation': 'identity', 'seal-exact-corpus-candidate': 'prepare', 'authenticate-corpus-preparation-provenance': 'authorize',
+  'protected-corpus-publisher': 'publish', 'customer-canary': 'canary', 'promote-canaried-corpus': 'promote' };
+export function failedStage(run) {
+  const recorded = Object.entries(run?.outcomeRecord?.jobs || {}).find(([, result]) => result === 'failure')?.[0];
+  const listed = (run?.jobs || []).find((job) => job.conclusion === 'failure' && JOB_NAMES[job.name]);
+  const stage = recorded || (listed ? JOB_NAMES[listed.name] : null);
+  return stage ? STAGE_NAMES[stage] || stage : `an unnamed job (conclusion ${run?.conclusion || 'unknown'})`;
+}
+
+/** A night's scheduled dispatches (the 07:36Z run and its retry slot) sit within this span; a day earlier is last night. */
+const NIGHT_SPAN_MS = 12 * HOUR;
+/**
+ * How many protected-release corpus runs belong to THIS night. Measured 2026-10-22: a manual workflow_dispatch proving
+ * run at 10:36Z Oct 21 and a stood-down retry slot at 11:25Z Oct 21 made ONE scheduled night read "attempt 2; 1 retried".
+ * The night is anchored on the newest `schedule` dispatcher that dispatched a run, plus scheduled dispatchers within
+ * NIGHT_SPAN_MS before it; a run counts only from that start and never when its dispatcher was a manual
+ * workflow_dispatch. A stood-down retry slot created no run, so it never counts. `runs` is newest-first, in window.
+ */
+export function nightAttempts(runs, dispatchers) {
+  const dispatchKey = (d) => `corpus-${d.id}-${d.attempt ?? 1}`;
+  const runKey = (run) => CORPUS_RUN_TITLE.exec(String(run.title || ''))?.[1];
+  const byKey = new Map(dispatchers.map((d) => [dispatchKey(d), d]));
+  const manual = (run) => byKey.get(runKey(run))?.event === 'workflow_dispatch';
+  const [newest] = runs;
+  if (manual(newest)) return ' (1 attempt; manual dispatch)';
+  const scheduled = dispatchers.filter((d) => !d.event || d.event === 'schedule'); // an undated event is read as the schedule
+  const anchor = scheduled.find((d) => runs.some((run) => runKey(run) === dispatchKey(d)));
+  const nightStart = anchor
+    ? Math.min(...scheduled.filter((d) => ms(d.createdAt) <= ms(anchor.createdAt) && ms(anchor.createdAt) - ms(d.createdAt) <= NIGHT_SPAN_MS).map((d) => ms(d.createdAt)))
+    : ms(newest.createdAt);
+  const count = runs.filter((run) => ms(run.createdAt) >= nightStart && !manual(run)).length || 1;
+  return count > 1 ? ` (attempt ${count} of the night; ${count - 1} retried)` : ' (1 attempt)';
+}
+
+/**
+ * THE ONE LINE the owner reads first: what the night did, result first, in Eastern time. `input` is
+ * gatherCorpusHealthInput's; the night is the newest corpus run of the last 24h and every retry of it.
+ * Never learned from a page: the watchdog writes this before ntfy ever fires.
+ */
+export function summarizeNight(input, now) {
+  const at = now instanceof Date ? now.getTime() : Number(now);
+  const inWindow = (iso) => { const t = ms(iso); return t !== null && t <= at && at - t <= TONIGHT_WINDOW_MS; };
+  const runs = (input?.corpusRuns || []).filter((run) => inWindow(run.createdAt))
+    .map((run) => ({ ...run, outcome: run.outcome ?? classifyCorpusRun(run) })).sort((a, b) => ms(b.createdAt) - ms(a.createdAt));
+  const dispatchers = (input?.dispatcherRuns || []).filter((d) => inWindow(d.createdAt)).sort((a, b) => ms(b.createdAt) - ms(a.createdAt));
+  const [dispatcher] = dispatchers;
+  const latest = input?.latestRelease;
+  const attempts = runs.length ? nightAttempts(runs, dispatchers) : '';
+  const keep = latest ? `customers keep ${latest.tag} (${hours(Math.max(0, at - (ms(latest.generation || latest.publishedAt) ?? at)))} old)` : 'no promoted generation is observed';
+  if (!runs.length) {
+    if (!dispatcher) return { outcome: 'no-run', line: `night: no corpus run in the last 24h (no dispatcher run either); ${keep}` };
+    if (dispatcher.status !== 'completed') return { outcome: 'dispatching', line: `night: dispatcher run ${dispatcher.id} is ${dispatcher.status} (started ${formatEastern(dispatcher.createdAt)})` };
+    if (dispatcher.conclusion !== 'success') return { outcome: 'dispatcher-failed', line: `night: dispatcher run ${dispatcher.id} concluded ${dispatcher.conclusion} at ${formatEastern(dispatcher.createdAt)}, nothing was dispatched; ${keep}` };
+    return { outcome: 'stood-down', line: `night: stood down at ${formatEastern(dispatcher.createdAt)} (CORPUS_NIGHTLY off, or no install-verified runtime); ${keep}` };
+  }
+  const [run] = runs;
+  const when = formatEastern(run.updatedAt || run.createdAt);
+  const degraded = input?.deferral?.observable && input.deferral.degraded && latest && input.deferral.newestTag === latest.tag
+    && (input.deferral.degraded.carried.length || input.deferral.degraded.missing.length) ? input.deferral.degraded : null;
+  switch (run.outcome) {
+    case 'published': {
+      const tag = latest && (ms(latest.publishedAt) ?? 0) >= (ms(run.createdAt) ?? 0) ? latest.tag : 'a new generation';
+      if (degraded) {
+        return { outcome: 'degraded', line: `night: degraded -- published ${tag} at ${when}${attempts} with ${degraded.carried.length} carried `
+          + `(${degraded.carried.join(', ') || 'none'}) and ${degraded.missing.length} missing (${degraded.missing.join(', ') || 'none'}), each named in CORPUS-COVERAGE.json` };
+      }
+      return { outcome: 'published', line: `night: published ${tag} at ${when}${attempts}` };
+    }
+    case 'no-change': return { outcome: 'no-change', line: `night: no-change at ${when}${attempts} -- every knowledge input equals the promoted generation; ${keep}` };
+    case 'superseded': return { outcome: 'superseded', line: `night: superseded at ${when} -- a newer code release landed; the next night builds at it; ${keep}` };
+    case 'canary-rejected': return { outcome: 'canary-rejected', line: `night: canary-rejected at ${when}${attempts} -- a clean customer install could not apply the candidate (GLOBAL: not retried); ${keep}` };
+    case 'in-progress': return { outcome: 'in-progress', line: `night: run ${run.id} still in progress, started ${formatEastern(run.createdAt)}${attempts}` };
+    case 'failed': return { outcome: 'failed', line: `night: failed in ${failedStage(run)} at ${when}${attempts}; ${keep}` };
+    default: return { outcome: 'unknown', line: `night: run ${run.id} succeeded at ${when} but reached no outcome${attempts}; ${keep}` };
+  }
+}
+
 export function renderReport(result, now) {
-  const lines = [`corpus-watchdog ${result.verdict} at ${now.toISOString()}`];
+  const lines = [`corpus-watchdog ${result.verdict} -- ${result.night?.line || `judged ${formatEastern(now)}`}`,
+    `  judged ${formatEastern(now)} (${now.toISOString()})`];
   for (const f of result.findings) lines.push(`  [${f.level}] ${f.check}: ${f.detail}`);
   return `${lines.join('\n')}\n`;
 }
@@ -380,7 +485,8 @@ async function main(argv = process.argv.slice(2)) {
   }
   let result;
   try {
-    result = judgeCorpusHealth(await gatherCorpusHealthInput({ repo, now }), now);
+    const input = await gatherCorpusHealthInput({ repo, now });
+    result = { ...judgeCorpusHealth(input, now), night: summarizeNight(input, now) };
   } catch (error) {
     // Could not observe = could not prove health. Absence of evidence is failure.
     process.stderr.write(`corpus-watchdog RED: evidence could not be gathered: ${error.message}\n`);

package/scripts/gist-receipts.mjs

@@ -34,6 +34,7 @@ import { canonicalJson, digest, validateGistAggregateReceipt } from './coverage-
 import { promoteArtifactSet } from '../kb/incremental-refresh.mjs';
 import { writeRvfGeneration } from './rvf-generation.mjs';
 import { fetchGistGitSnapshot } from './gist-git-transport.mjs';
+import { isTransientText, retryTransientAsync } from './transient-gh.mjs';
 
 const DEFAULT_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
 const TEXT_EXT = new Set(['.md', '.markdown', '.txt', '.rst']);
@@ -402,9 +403,15 @@ function reusableCachedGist(cached, stub) {
   return 'reuse';
 }
 
+// The Git snapshot is one `ls-remote` + one bare clone per gist against gist.github.com. A transient transport
+// failure (reset, timeout, "could not read Username", or the gist moving mid-capture) is retried, bounded, so
+// one blip on one of ~180 gists cannot fail the aggregate and with it the night. A gist that is gone
+// ("repository not found") is an answer and is thrown at once.
+const transientGistSnapshot = (error) => isTransientText(error?.message) || /moved while Git snapshot was captured/.test(String(error?.message || ''));
+
 export async function captureGistSources({ observation, cache = null, fetchDetail = null,
   fetchRaw = defaultFetchRaw, fetchGitSnapshot = fetchGistGitSnapshot, signal,
-  now = () => new Date().toISOString() } = {}) {
+  now = () => new Date().toISOString(), snapshotRetryDelayMs = 15_000 } = {}) {
   const stubs = observation?.gists?.rows;
   if (!Array.isArray(stubs) || stubs.some(({ id }) => !HEX_GIST.test(String(id || '')))
     || new Set(stubs.map(({ id }) => id)).size !== stubs.length) {
@@ -420,7 +427,8 @@ export async function captureGistSources({ observation, cache = null, fetchDetai
       // The production/no-key route never spends one REST detail request per gist and never trusts
       // a body-free cache or timestamps as content identity. A full bare clone gives one immutable
       // Git commit/tree; every observed raw_url's blob identity must match its HEAD file bytes.
-      const snapshot = await fetchGitSnapshot({ owner: observation.owner, stub, signal });
+      const snapshot = await retryTransientAsync(() => fetchGitSnapshot({ owner: observation.owner, stub, signal }),
+        { isTransient: transientGistSnapshot, baseDelayMs: snapshotRetryDelayMs });
       const files = [];
       for (const source of snapshot.files) {
         const filename = source.filename;

package/scripts/release.mjs

@@ -71,6 +71,7 @@ import {
 import { bindCoverageToReceipt, writeCoverageAssets } from './corpus-coverage-sidecar.mjs';
 import { describeDegraded } from './corpus-store-failure.mjs';
 import { assertNoNewerCorpusGeneration } from './code-release-corpus.mjs';
+import { retryingGh } from './transient-gh.mjs';
 
 const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
 const PUBLISH = process.argv.includes('--publish');
@@ -141,11 +142,17 @@ const compareCodeTags = (left, right) => {
   return Math.sign(a[0] - b[0] || a[1] - b[1] || a[2] - b[2]);
 };
 
-/** The one gh seam both corpus entry points use (RUVNET_GH_COMMAND / RUVNET_GH_SCRIPT for tests). */
+/**
+ * The one gh seam both corpus entry points use (RUVNET_GH_COMMAND / RUVNET_GH_SCRIPT for tests). Reads and the
+ * idempotent `release edit` retry a transient failure, bounded (scripts/transient-gh.mjs): on 2026-09-29 the
+ * night was red on "cannot confirm the promoted corpus release" -- one `release view` that failed right after
+ * the write had succeeded. `release create` runs exactly once; the absence proof above it is what makes a
+ * repeat safe to refuse.
+ */
 function corpusGh(env, run) {
-  const ghCommand = env.RUVNET_GH_COMMAND || 'gh';
-  const ghPrefix = env.RUVNET_GH_SCRIPT ? [env.RUVNET_GH_SCRIPT] : [];
-  const gh = (args) => run(ghCommand, [...ghPrefix, ...args], { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
+  const gh = retryingGh({ run, command: env.RUVNET_GH_COMMAND || 'gh', prefix: env.RUVNET_GH_SCRIPT ? [env.RUVNET_GH_SCRIPT] : [],
+    options: { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] },
+    onRetry: ({ attempt, args, delayMs }) => console.error(`[corpus-seed] gh ${args.slice(0, 2).join(' ')} failed transiently (attempt ${attempt}); retrying in ${delayMs / 1000}s`) });
   const ghJson = (args, label) => {
     const result = gh(args);
     if (result.error || result.status !== 0) {

package/scripts/source-coverage.mjs

@@ -9,6 +9,7 @@ import { readRvfGenerations, sha256File } from './rvf-generation.mjs';
 import { canonicalJson, coverageGenerationFor, digest, validateGistAggregateReceipt } from './coverage-integrity.mjs';
 import { repositoryNames } from '../kb/card-lane.mjs';
 import { rootNeverMaterialized, storeRoot } from '../kb/store-root.mjs';
+import { isTransientText, retryTransient, sleepSync } from './transient-gh.mjs';
 
 export { canonicalJson, digest } from './coverage-integrity.mjs';
 
@@ -52,6 +53,11 @@ function runGh(args) {
   return result.stdout;
 }
 
+// Every observation read below goes through this: a transient failure (rate limit, 5xx, a reset) is retried,
+// bounded, before it can fail the night (scripts/transient-gh.mjs). Applied around the INJECTED `gh`, not
+// inside runGh, so the fault-injection suite exercises the same retry the nightly runs.
+const retryingRead = (gh, sleep) => (args) => retryTransient(() => gh(args), { sleep });
+
 // A fork's disposition depends on how many commits it carries that its upstream does not. The
 // compare is pinned to the two heads THIS observation recorded (never to branch names, which move),
 // and is run against the upstream repository in GitHub's documented cross-fork form
@@ -80,7 +86,8 @@ function observeForkDelta(owner, node, gh) {
   };
 }
 
-export function observeRepositories(owner, { gh = runGh } = {}) {
+export function observeRepositories(owner, { gh: rawGh = runGh, sleep = sleepSync } = {}) {
+  const gh = retryingRead(rawGh, sleep);
   const rows = [];
   const pages = [];
   let cursor = '';
@@ -103,10 +110,11 @@ export function observeRepositories(owner, { gh = runGh } = {}) {
   return { rows: withDeltas, expected, pages };
 }
 
-// The row an external source reports once its upstream is gone: exactly what was last observed, so the
-// stored content classifies CURRENT, nothing is re-ingested, and the row says plainly it is no longer live.
-function goneExternalRow(source) {
-  const gone = source.ifGone;
+// The row an external source reports when its upstream cannot be read: exactly what was last observed (or last
+// ingested), so the stored content classifies CURRENT, nothing is re-ingested, and the row says plainly why.
+// `gone` is the policy's ifGone entry (404/410); for a source unreadable for a moment it is the seed ledger's
+// own commit, with `reason` naming the failure.
+function unavailableExternalRow(source, gone, reason) {
   if (!/^[0-9a-f]{40}$/i.test(String(gone.sourceCommit || ''))) {
     throw new Error(`external source ${source.repository}: ifGone.sourceCommit must be the 40-hex commit last ingested`);
   }
@@ -114,12 +122,27 @@ function goneExternalRow(source) {
     name: source.repository.split('/')[1], fullName: source.repository, storeName: source.store,
     url: `https://github.com/${source.repository}`, description: gone.description ?? null, homepageUrl: null,
     isFork: false, upstreamIsFork: false, isArchived: false, isDisabled: false,
-    updatedAt: gone.updatedAt ?? null, pushedAt: gone.pushedAt ?? null, upstreamUnavailable: true,
+    updatedAt: gone.updatedAt ?? null, pushedAt: gone.pushedAt ?? null, upstreamUnavailable: true, upstreamUnavailableReason: reason,
     defaultBranchRef: { name: gone.defaultBranch || 'HEAD', target: { oid: gone.sourceCommit, committedDate: gone.committedAt ?? null } },
   };
 }
 
-export function observeExternalRepositories(sources, { gh = runGh } = {}) {
+/** The one reason a transient carry records; scripts/corpus-reconcile.mjs refuses a second consecutive one by it. */
+export const TRANSIENT_UNREADABLE_REASON = 'was unreadable this observation (transient failure after the bounded retry); it is read again next night';
+/** The carry record a transiently unreadable source ships: STALE, not CURRENT, and counted toward the degraded bound. */
+export const TRANSIENT_CARRY_REASON = 'transient: upstream unreadable';
+
+/**
+ * `carriedCommits`: { <store>: { sourceCommit, committedAt } } from the seed's own generation ledger, already stripped
+ * of any store the seed's coverage shows was ALREADY carried this way (corpus-reconcile.mjs transientCarryCommits),
+ * so the same source is never carried two nights running. A source whose read still fails TRANSIENTLY after the
+ * bounded retry (rate limit, 5xx, a reset; measured 2026-09-26) is pinned at that commit for ONE night: its row
+ * classifies STALE with a `carry` record (nobody read upstream tonight, so it is not called current), it counts
+ * toward max(3, 5% of eligible) with every other carried store, and the next night reads it again. Without a prior
+ * commit there is nothing honest to carry, and the observation fails.
+ */
+export function observeExternalRepositories(sources, { gh: rawGh = runGh, sleep = sleepSync, carriedCommits = null } = {}) {
+  const gh = retryingRead(rawGh, sleep);
   const rows = [];
   const pages = [];
   for (const source of sources) {
@@ -134,13 +157,28 @@ export function observeExternalRepositories(sources, { gh = runGh } = {}) {
         ? JSON.parse(gh(['api', `repos/${source.repository}/commits/${repo.default_branch}`]))
         : null;
     } catch (error) {
-      // An upstream that GitHub reports gone (404/410) must not stop every night for every customer.
-      // Only a source whose policy entry records `ifGone` (the commit it was last ingested at) may be
-      // carried; anything else -- 403, rate limit, a mistyped repository -- still fails the observation.
-      if (!source.ifGone || !/\(HTTP (404|410)\)/.test(String(error.message))) throw error;
-      process.stderr.write(`[source-coverage] external source ${source.repository} is gone upstream; carrying the last ingested content (${source.ifGone.sourceCommit})\n`);
-      rows.push(goneExternalRow(source));
-      pages.push({ repository: source.repository, responseDigest: digest({ gone: source.repository, ifGone: source.ifGone }), count: 1, terminal: true });
+      // An upstream that GitHub reports gone (404/410) must not stop every night for every customer: a source
+      // whose policy entry records `ifGone` (the commit it was last ingested at) is carried there. A source that
+      // is merely unreadable tonight (still transient after the retry) is carried at the seed's own commit when
+      // it has one. Anything else -- a bare 403, a mistyped repository, no prior bytes -- still fails the observation.
+      const message = String(error.message);
+      const gone = source.ifGone && /\(HTTP (404|410)\)/.test(message);
+      const prior = carriedCommits?.[String(source.store).toLowerCase()];
+      const carried = String(prior?.sourceCommit ?? prior ?? '').toLowerCase();
+      const unreadable = !gone && isTransientText(message) && /^[0-9a-f]{40}$/.test(carried);
+      if (!gone && !unreadable) throw error;
+      // FIXED shapes, never the error text or a timestamp: the observation digest feeds the no-change decision,
+      // so a night that carried one unreadable source and saw nothing else move must still read as unchanged.
+      const kind = gone ? 'gone' : 'transient';
+      const reason = gone ? 'is no longer reachable (HTTP 404/410)' : TRANSIENT_UNREADABLE_REASON;
+      // committedAt comes from the seed's own coverage row (corpus-reconcile.mjs carriedCommittedAt), so the customer's
+      // "oldest carried" line dates the pinned commit instead of showing an unknown date.
+      const at = gone ? source.ifGone : { sourceCommit: carried, committedAt: prior?.committedAt ?? null };
+      process.stderr.write(`[source-coverage] external source ${source.repository} ${reason}; carrying the last ingested content (${at.sourceCommit})`
+        + `${gone ? '' : ` -- ${message.replace(/\s+/g, ' ').trim().slice(0, 200)}`}\n`);
+      rows.push({ ...unavailableExternalRow(source, at, reason), upstreamUnavailableKind: kind });
+      pages.push({ repository: source.repository, responseDigest: digest(gone ? { gone: source.repository, ifGone: source.ifGone }
+        : { unavailable: source.repository, kind, at: carried }), count: 1, terminal: true });
       continue;
     }
     const row = {
@@ -164,9 +202,11 @@ export function observeSourceUniverse({
   externalSources = [],
   gh = runGh,
   observedAt = new Date().toISOString(),
+  sleep = sleepSync,
+  carriedCommits = null,
 } = {}) {
-  const primary = observeRepositories(owner, { gh });
-  const external = observeExternalRepositories(externalSources, { gh });
+  const primary = observeRepositories(owner, { gh, sleep });
+  const external = observeExternalRepositories(externalSources, { gh, sleep, carriedCommits });
   const repositoryRows = canonicalRepositoryRows([...primary.rows, ...external.rows]);
   const repositories = sourceSetObservation(repositoryRows, primary.expected + external.expected);
   const repositoryKeys = new Set();
@@ -179,7 +219,7 @@ export function observeSourceUniverse({
     repositoryKeys.add(repositoryKey);
     storeKeys.add(storeKey);
   }
-  const observedGists = observeGists(owner, { gh });
+  const observedGists = observeGists(owner, { gh, sleep });
   const gists = sourceSetObservation(canonicalGistRows(observedGists.rows), observedGists.expected);
   return canonicalSourceObservation({
     schemaVersion: 1,
@@ -222,6 +262,8 @@ export function canonicalRepositoryRows(rows = []) {
     isArchived: repo?.isArchived === true,
     isDisabled: repo?.isDisabled === true,
     ...(repo?.upstreamUnavailable === true ? { upstreamUnavailable: true } : {}),
+    ...(repo?.upstreamUnavailable === true && repo?.upstreamUnavailableReason ? { upstreamUnavailableReason: String(repo.upstreamUnavailableReason) } : {}),
+    ...(repo?.upstreamUnavailable === true && repo?.upstreamUnavailableKind ? { upstreamUnavailableKind: String(repo.upstreamUnavailableKind) } : {}),
     diskUsage: repo?.diskUsage,
     updatedAt: repo?.updatedAt ?? null,
     pushedAt: repo?.pushedAt ?? null,
@@ -325,7 +367,11 @@ function listGistsUnauthenticated(owner, curl) {
   return pages;
 }
 
-export function observeGists(owner, { gh = runGh, curl = runCurl } = {}) {
+export function observeGists(owner, { gh: rawGh = runGh, curl: rawCurl = runCurl, sleep = sleepSync } = {}) {
+  // A rate limit on either channel (measured 2026-09-26: the gist list) is retried, bounded, before the
+  // integration-token 403 is allowed to fall back to the anonymous list, which is itself rate-limited.
+  const gh = retryingRead(rawGh, sleep);
+  const curl = retryingRead(rawCurl, sleep);
   let pages;
   try {
     const raw = gh(['api', `users/${owner}/gists?per_page=100`, '--paginate', '--slurp']);
@@ -414,8 +460,19 @@ export function classifyRepository(repo, evidence, exclusion = null, outcome = n
     status = 'FAILED';
     reasons.push(outcome.integrity);
   }
-  if (repo.upstreamUnavailable === true) reasons.push(`upstream ${repo.fullName} is no longer reachable (HTTP 404/410); serving the content last ingested at ${upstreamSha}`);
+  if (repo.upstreamUnavailable === true) {
+    reasons.push(`upstream ${repo.fullName} ${repo.upstreamUnavailableReason || 'is no longer reachable (HTTP 404/410)'}; serving the content last ingested at ${upstreamSha}`);
+  }
   const record = {};
+  // A source nobody could read tonight is never CURRENT: its bytes are pinned at the last commit the seed carried,
+  // and the row ships a carry record (missedUpstream null -- the upstream head is unknown, not merely different)
+  // that eligibleRepositoryStanding accepts only together with upstreamUnavailable. A `gone` source stays CURRENT:
+  // its upstream was READ and is final.
+  if (disposed && repo.upstreamUnavailableKind === 'transient' && status === 'CURRENT') {
+    status = 'STALE';
+    record.carry = { reason: TRANSIENT_CARRY_REASON, carriedSourceCommit: upstreamSha, missedUpstream: null,
+      attempts: 1, carriedCommittedAt: repo.defaultBranchRef?.target?.committedDate || null };
+  }
   if (disposed && outcome?.carry && status === 'STALE') record.carry = { ...outcome.carry };
   if (disposed && outcome?.failure && status === 'MISSING') record.failure = { ...outcome.failure };
   return {
@@ -428,6 +485,7 @@ export function classifyRepository(repo, evidence, exclusion = null, outcome = n
     disposition,
     archived: repo.isArchived === true,
     ...(repo.upstreamUnavailable === true ? { upstreamUnavailable: true } : {}),
+    ...(repo.upstreamUnavailable === true && repo.upstreamUnavailableKind ? { upstreamUnavailableKind: repo.upstreamUnavailableKind } : {}),
     ...(repo.forkDelta ? { forkDelta: { ...repo.forkDelta } } : {}),
     upstream: { sha: upstreamSha, committedAt: repo.defaultBranchRef?.target?.committedDate || null,
       pushedAt: repo.pushedAt, updatedAt: repo.updatedAt },