ruvnet-brain 4.7.15 → 4.7.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/package.json +1 -1
- package/plugin/.claude-plugin/plugin.json +1 -1
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/scripts/coverage-integrity.mjs +9 -3
- package/scripts/approved-runtime.mjs +6 -2
- package/scripts/corpus-canary.mjs +14 -8
- package/scripts/corpus-dispatch-decision.mjs +56 -2
- package/scripts/corpus-next-seed.mjs +5 -1
- package/scripts/corpus-reconcile.mjs +39 -4
- package/scripts/corpus-watchdog.mjs +89 -9
- package/scripts/gist-receipts.mjs +10 -2
- package/scripts/release.mjs +11 -4
- package/scripts/source-coverage.mjs +77 -19
- package/scripts/transient-gh.mjs +150 -0
package/README.md
CHANGED
|
@@ -9,7 +9,7 @@ Created: 2026-06-29 22:36:38 EDT
|
|
|
9
9
|
|
|
10
10
|
### Discover the stack. Read the source. Build with confidence.
|
|
11
11
|
|
|
12
|
-
[](plugin/.claude-plugin/plugin.json)
|
|
13
13
|
|
|
14
14
|
**A source-grounded companion for Claude Code and Codex that helps you find and use Reuven Cohen's (rUv's) RuvNet ecosystem.**
|
|
15
15
|
|
|
@@ -255,7 +255,7 @@ uses the protected release workflow; a consumer update downloads an available ge
|
|
|
255
255
|
does not rebuild upstream knowledge. Check `--doctor`, `SOURCE.json` and refresh receipts
|
|
256
256
|
before treating an installation as current.
|
|
257
257
|
|
|
258
|
-
The **4.6.0 published-corpus baseline** is **203 public stores · 164,
|
|
258
|
+
The **4.6.0 published-corpus baseline** is **203 public stores · 164,863 public source chunks**.
|
|
259
259
|
Those are bundle census values, not a claim that every upstream repository is fresh today.
|
|
260
260
|
Later corpus publication may advance the knowledge generation independently of code changes.
|
|
261
261
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ruvnet-brain",
|
|
3
|
-
"version": "4.7.
|
|
3
|
+
"version": "4.7.16",
|
|
4
4
|
"description": "One-command installer for RuvNet Brain — a portable, source-grounded brain over rUv's RuvNet building blocks, delivered as a Claude Code plugin so Claude uses the stack instead of fighting it.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ruvnet-brain",
|
|
3
3
|
"description": "RuvNet brain transplant for Claude Code — grounds every RuvNet decision in real source across 77 rUv repositories, prefers Ruflo / RuVector-RVF / AgentDB over training-prior defaults (pgvector, Pinecone, hand-rolled cosine), and can pull in any RuvNet repo on demand. Ships a UserPromptSubmit retrieve-and-inject grounding hook and a PreToolUse write gate that refuses ungrounded rUv-product code until search_ruvnet has been consulted (ADR-0012 / ADR-067).",
|
|
4
|
-
"version": "4.7.
|
|
4
|
+
"version": "4.7.16",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "Stuart Kerr"
|
|
7
7
|
},
|
|
@@ -293,11 +293,17 @@ export function eligibleRepositoryStanding(row) {
|
|
|
293
293
|
if (status === 'CURRENT') return carry === undefined && failure === undefined ? 'shipped' : null;
|
|
294
294
|
if (status === 'STALE' && carry && typeof carry === 'object' && failure === undefined) {
|
|
295
295
|
const carried = String(carry.carriedSourceCommit || '').toLowerCase();
|
|
296
|
+
const upstream = String(row.upstream?.sha || '').toLowerCase();
|
|
297
|
+
// Two honest shapes. A refresh that FAILED: upstream was read, differs from the carried commit, and is named
|
|
298
|
+
// as missedUpstream. An upstream nobody could READ tonight (transient, source-coverage.mjs): the row is pinned
|
|
299
|
+
// at the carried commit, says `upstreamUnavailable`, and missedUpstream is null -- unknown, never invented.
|
|
300
|
+
const missed = String(carry.missedUpstream || '').toLowerCase();
|
|
301
|
+
const upstreamKnown = HEX40.test(missed) && missed === upstream && carried !== missed;
|
|
302
|
+
const upstreamUnread = carry.missedUpstream === null && row.upstreamUnavailable === true && row.upstreamUnavailableKind === 'transient'
|
|
303
|
+
&& carry.reason === 'transient: upstream unreadable' && carried === upstream;
|
|
296
304
|
const shipped = typeof carry.reason === 'string' && carry.reason.length > 0
|
|
297
305
|
&& HEX40.test(carried) && carried === String(row.artifact?.sourceCommit || '').toLowerCase()
|
|
298
|
-
&&
|
|
299
|
-
&& String(carry.missedUpstream).toLowerCase() === String(row.upstream?.sha || '').toLowerCase()
|
|
300
|
-
&& carried !== String(carry.missedUpstream).toLowerCase()
|
|
306
|
+
&& (upstreamKnown || upstreamUnread)
|
|
301
307
|
&& isAttemptCount(carry.attempts) && isIsoOrNull(carry.carriedCommittedAt)
|
|
302
308
|
&& row.artifact?.bytesVerified === true && row.artifact?.passagesPresent === true;
|
|
303
309
|
return shipped ? 'shipped' : null;
|
|
@@ -46,6 +46,7 @@ import fs from 'node:fs';
|
|
|
46
46
|
import os from 'node:os';
|
|
47
47
|
import path from 'node:path';
|
|
48
48
|
import { spawnSync } from 'node:child_process';
|
|
49
|
+
import { retryTransientResult } from './transient-gh.mjs';
|
|
49
50
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
50
51
|
|
|
51
52
|
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
@@ -190,9 +191,12 @@ const semverDescending = (left, right) => {
|
|
|
190
191
|
return b[0] - a[0] || b[1] - a[1] || b[2] - a[2];
|
|
191
192
|
};
|
|
192
193
|
|
|
194
|
+
// Every read here is a `gh` read of public release metadata or assets; a transient failure (rate limit, 5xx,
|
|
195
|
+
// a reset mid-download of the ~555 MB archive) is retried, bounded, before it can fail the dispatcher, the
|
|
196
|
+
// identity job or the publisher (scripts/transient-gh.mjs). A refused aggregate is an answer and is never retried.
|
|
193
197
|
function defaultGh(args) {
|
|
194
|
-
const result = spawnSync(process.env.RUVNET_GH_COMMAND || 'gh', args,
|
|
195
|
-
{ encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, timeout: 15 * 60_000 });
|
|
198
|
+
const result = retryTransientResult(() => spawnSync(process.env.RUVNET_GH_COMMAND || 'gh', args,
|
|
199
|
+
{ encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, timeout: 15 * 60_000 }));
|
|
196
200
|
if (result.error || result.status !== 0) {
|
|
197
201
|
throw new Error(`gh ${args.slice(0, 3).join(' ')} failed: ${String(result.error?.message || result.stderr || `exit ${result.status}`).trim().slice(0, 400)}`);
|
|
198
202
|
}
|
|
@@ -47,6 +47,7 @@ import { addPrivateStore, resolveRuntime, resultFromRefreshReceipt, runInstaller
|
|
|
47
47
|
import { CANARY_VERDICT_KIND, CORPUS_TAG_PATTERN, REQUIRED_CANARY_CHECKS } from './corpus-promotion.mjs';
|
|
48
48
|
import { inventoryFootprint } from '../plugin/scripts/brain-footprint.mjs';
|
|
49
49
|
import { eligibleRepositoryStanding } from '../plugin/scripts/coverage-integrity.mjs';
|
|
50
|
+
import { TRANSIENT_HTTP, retryTransientAsync } from './transient-gh.mjs';
|
|
50
51
|
|
|
51
52
|
export const PUBLIC_API = 'https://api.github.com';
|
|
52
53
|
export const FRESHNESS_LIMIT_MS = 48 * 3_600_000;
|
|
@@ -151,9 +152,11 @@ export function storeFreshness({ after, before = null, now }) {
|
|
|
151
152
|
const upstream = row.upstream?.sha || null;
|
|
152
153
|
const built = row.artifact?.sourceCommit || null;
|
|
153
154
|
if (!upstream || !built) continue;
|
|
155
|
+
// Named either way: a refresh that failed (upstream read, differs) or an upstream nobody could read tonight
|
|
156
|
+
// (pinned at the carried commit, `upstreamUnavailable`). Both are carried with a record the validator accepts.
|
|
157
|
+
if (row.kind === 'repository' && row.carry && eligibleRepositoryStanding(row) === 'shipped') { carried.push(row.artifact?.store || row.key); continue; }
|
|
154
158
|
if (upstream !== built) {
|
|
155
|
-
|
|
156
|
-
else stale.push(row.artifact?.store || row.key);
|
|
159
|
+
stale.push(row.artifact?.store || row.key);
|
|
157
160
|
continue;
|
|
158
161
|
}
|
|
159
162
|
if (before && priorCommit.get(row.key) !== upstream) moved += 1;
|
|
@@ -285,15 +288,18 @@ function runLogged(command, args, { env, cwd, timeoutMs, log }) {
|
|
|
285
288
|
return result;
|
|
286
289
|
}
|
|
287
290
|
|
|
288
|
-
/**
|
|
289
|
-
|
|
291
|
+
/**
|
|
292
|
+
* Phase 1: the approved runtime, installed exactly as `npx ruvnet-brain@<version>` would. The registry read is
|
|
293
|
+
* retried on a transient failure (bounded) so a registry blip reads as weather, never as "the customer refused".
|
|
294
|
+
*/
|
|
295
|
+
export async function installCustomer({ approvedVersion, work, home, log = () => {}, retryDelayMs = 30_000 }) {
|
|
290
296
|
const env = customerEnv({ home, work });
|
|
291
297
|
fs.mkdirSync(env.TMPDIR, { recursive: true });
|
|
292
298
|
const prefix = path.join(work, 'npx');
|
|
293
299
|
fs.mkdirSync(prefix, { recursive: true });
|
|
294
300
|
const npm = path.join(path.dirname(process.execPath), process.platform === 'win32' ? 'npm.cmd' : 'npm');
|
|
295
|
-
runLogged(npm, ['install', '--prefix', prefix, '--no-audit', '--no-fund', '--no-save', `ruvnet-brain@${approvedVersion}`],
|
|
296
|
-
{ env, cwd: prefix, timeoutMs: 600_000, log });
|
|
301
|
+
await retryTransientAsync(() => runLogged(npm, ['install', '--prefix', prefix, '--no-audit', '--no-fund', '--no-save', `ruvnet-brain@${approvedVersion}`],
|
|
302
|
+
{ env, cwd: prefix, timeoutMs: 600_000, log }), { baseDelayMs: retryDelayMs, onRetry: ({ attempt }) => log(`--- npm install failed transiently (attempt ${attempt}); retrying`) });
|
|
297
303
|
const installer = path.join(prefix, 'node_modules', 'ruvnet-brain', 'bin', 'install.mjs');
|
|
298
304
|
const installed = readJson(path.join(prefix, 'node_modules', 'ruvnet-brain', 'package.json')).version;
|
|
299
305
|
if (installed !== approvedVersion) throw new Error(`npm served ruvnet-brain ${installed}, not the approved ${approvedVersion}`);
|
|
@@ -316,8 +322,8 @@ function runUpdater({ kbDir, env, resultFile }) {
|
|
|
316
322
|
}
|
|
317
323
|
|
|
318
324
|
// An anonymous GitHub API read from a shared runner address is sometimes refused (403/429) or fails (5xx)
|
|
319
|
-
// for a moment; one such blip must not fail the night. The same call is retried after a short wait.
|
|
320
|
-
|
|
325
|
+
// for a moment; one such blip must not fail the night. The same call is retried after a short wait. The
|
|
326
|
+
// status set is the pipeline's one definition (scripts/transient-gh.mjs TRANSIENT_HTTP), not a private copy.
|
|
321
327
|
async function fetchJson(url, what, { retryDelayMs = 15_000, maxAttempts = 4 } = {}) {
|
|
322
328
|
for (let attempt = 1; ; attempt += 1) {
|
|
323
329
|
let status = null;
|
|
@@ -19,11 +19,31 @@
|
|
|
19
19
|
// the new workflow passes (--approved-tag, --no-change-out, --stage-candidate, --promote-staged), so the nightly stands down rather than
|
|
20
20
|
// let new workflow text drive old scripts.
|
|
21
21
|
|
|
22
|
+
//
|
|
23
|
+
// THE RETRY SLOT (2026-10-20). Measured on 38 nights: 24 failed, most on one transient blip, and nothing
|
|
24
|
+
// retried until the next calendar day. The dispatcher now also fires at RETRY_CRON (11:17 UTC, four hours
|
|
25
|
+
// after the night's first slot) and asks decideRetry(): re-dispatch ONLY when every corpus run of the
|
|
26
|
+
// night so far ended `failed` (or reached no outcome), none is still running, none ended published /
|
|
27
|
+
// no-change / superseded / canary-rejected (those are answers, not weather), and fewer than
|
|
28
|
+
// MAX_CORPUS_RUNS_PER_NIGHT runs exist -- so one night gets at most one retry, and a deterministic
|
|
29
|
+
// failure fails closed a second time rather than looping. The kill switch and the approved-runtime
|
|
30
|
+
// resolution still run after it, exactly as on the first slot. Double publication is impossible by
|
|
31
|
+
// construction: a failed run promoted nothing, every generation is content-addressed, and release.mjs
|
|
32
|
+
// refuses a tag that already exists.
|
|
33
|
+
|
|
22
34
|
import fs from 'node:fs';
|
|
23
35
|
import path from 'node:path';
|
|
24
36
|
import { pathToFileURL } from 'node:url';
|
|
25
37
|
|
|
26
38
|
export const MIN_PIPELINE_VERSION = '4.3.38'; // sync-version-ignore: a fixed pipeline floor, not the current version
|
|
39
|
+
/** The dispatcher's second schedule entry; corpus-nightly-dispatch.yml's `on.schedule` must carry exactly this line. */
|
|
40
|
+
export const RETRY_CRON = '17 11 * * *';
|
|
41
|
+
/** How far back the retry slot looks for the night's runs: past the 07:17 UTC first slot, never into yesterday. */
|
|
42
|
+
export const RETRY_WINDOW_MS = 10 * 3_600_000;
|
|
43
|
+
export const MAX_CORPUS_RUNS_PER_NIGHT = 2;
|
|
44
|
+
// `unknown` (every job green, no outcome) is retried only when the run itself did NOT conclude success: a green
|
|
45
|
+
// run that reached no outcome is a workflow defect the terminal-outcome job already fails loudly, not weather.
|
|
46
|
+
const retryable = (run) => run.outcome === 'failed' || (run.outcome === 'unknown' && run.conclusion !== 'success');
|
|
27
47
|
const SEMVER = /^(\d+)\.(\d+)\.(\d+)$/;
|
|
28
48
|
const HEX40 = /^[0-9a-f]{40}$/;
|
|
29
49
|
const CODE_TAG = /^v\d+\.\d+\.\d+$/;
|
|
@@ -82,6 +102,21 @@ export function decide({ nightlyVar, resolution, minPipelineVersion = MIN_PIPELI
|
|
|
82
102
|
return { verdict: 'arm', reason: `building the corpus at approved runtime ${tag} @ ${sourceSha}`, tag, version, sourceSha };
|
|
83
103
|
}
|
|
84
104
|
|
|
105
|
+
/**
|
|
106
|
+
* The retry slot's one decision. `runs`: the night's protected-release corpus runs so far
|
|
107
|
+
* (scripts/corpus-watchdog.mjs gatherCorpusRuns, each with `outcome` from classifyCorpusRun).
|
|
108
|
+
* Returns { verdict: 'retry' | 'stand-down', reason }.
|
|
109
|
+
*/
|
|
110
|
+
export function decideRetry({ runs = [], maxRuns = MAX_CORPUS_RUNS_PER_NIGHT } = {}) {
|
|
111
|
+
if (!runs.length) return { verdict: 'stand-down', reason: 'the first slot dispatched no corpus run tonight (it stood down or failed); the watchdog judges that, a retry would not' };
|
|
112
|
+
const running = runs.find((run) => run.outcome === 'in-progress');
|
|
113
|
+
if (running) return { verdict: 'stand-down', reason: `corpus run ${running.id} is still in progress` };
|
|
114
|
+
const settled = runs.find((run) => !retryable(run));
|
|
115
|
+
if (settled) return { verdict: 'stand-down', reason: `corpus run ${settled.id} ended ${settled.outcome}; that is the night's answer, not weather` };
|
|
116
|
+
if (runs.length >= maxRuns) return { verdict: 'stand-down', reason: `${runs.length} corpus run(s) already failed tonight; the bound is ${maxRuns} per night, so this is not retried again` };
|
|
117
|
+
return { verdict: 'retry', reason: `corpus run ${runs.map((run) => run.id).join(', ')} failed and nothing published; dispatching the night's one retry` };
|
|
118
|
+
}
|
|
119
|
+
|
|
85
120
|
/** approved-runtime.mjs --resolve's exit status + stdout (+ stderr, naming a refused release) -> a resolution. */
|
|
86
121
|
export function resolutionFrom({ status, stdout, stderr = '' }) {
|
|
87
122
|
if (String(status) === '3') return { status: 'not-yet-verified' };
|
|
@@ -107,12 +142,31 @@ const arg = (argv, name) => {
|
|
|
107
142
|
* CLI: --nightly <value> --resolve-status <n|skipped> --release <file> [--resolve-stderr <file>]
|
|
108
143
|
* --github-output <file>. Exit 1 only on `fail`.
|
|
109
144
|
* --is-off <value> prints true|false (lets the workflow skip the ~555 MB resolve when the switch is off)
|
|
145
|
+
* --retry-slot --repo <owner/name> [--now ISO] --github-output <file>
|
|
146
|
+
* writes retry=true|false and retry_reason; exit 1 only when run history could not be read
|
|
110
147
|
*/
|
|
111
|
-
export function main(argv = process.argv.slice(2), { stdout = process.stdout } = {}) {
|
|
148
|
+
export async function main(argv = process.argv.slice(2), { stdout = process.stdout } = {}) {
|
|
112
149
|
if (argv.includes('--is-off')) {
|
|
113
150
|
stdout.write(`${killSwitchOff(arg(argv, '--is-off'))}\n`);
|
|
114
151
|
return 0;
|
|
115
152
|
}
|
|
153
|
+
if (argv.includes('--retry-slot')) {
|
|
154
|
+
const { classifyCorpusRun, gatherCorpusRuns } = await import('./corpus-watchdog.mjs');
|
|
155
|
+
const now = arg(argv, '--now') ? Date.parse(arg(argv, '--now')) : Date.now();
|
|
156
|
+
const repo = arg(argv, '--repo') || process.env.GITHUB_REPOSITORY;
|
|
157
|
+
let decision;
|
|
158
|
+
try {
|
|
159
|
+
const runs = gatherCorpusRuns({ repo, since: now - RETRY_WINDOW_MS }).map((run) => ({ ...run, outcome: classifyCorpusRun(run) }));
|
|
160
|
+
decision = decideRetry({ runs });
|
|
161
|
+
} catch (error) {
|
|
162
|
+
stdout.write(`::error::corpus-nightly-dispatch retry slot could not read the night's runs: ${error.message}\n`);
|
|
163
|
+
return 1;
|
|
164
|
+
}
|
|
165
|
+
const githubOutput = arg(argv, '--github-output');
|
|
166
|
+
if (githubOutput) fs.appendFileSync(path.resolve(githubOutput), `retry=${decision.verdict === 'retry'}\nretry_reason=${decision.reason.replace(/\n/g, ' ')}\n`);
|
|
167
|
+
stdout.write(`::notice::corpus-nightly-dispatch retry slot ${decision.verdict}: ${decision.reason}\n`);
|
|
168
|
+
return 0;
|
|
169
|
+
}
|
|
116
170
|
const nightlyVar = arg(argv, '--nightly') ?? '';
|
|
117
171
|
const status = arg(argv, '--resolve-status');
|
|
118
172
|
const releaseFile = arg(argv, '--release');
|
|
@@ -134,5 +188,5 @@ export function main(argv = process.argv.slice(2), { stdout = process.stdout } =
|
|
|
134
188
|
}
|
|
135
189
|
|
|
136
190
|
if (process.argv[1] && pathToFileURL(path.resolve(process.argv[1])).href === import.meta.url) {
|
|
137
|
-
process.exitCode =
|
|
191
|
+
main().then((code) => { process.exitCode = code; });
|
|
138
192
|
}
|
|
@@ -61,6 +61,7 @@ import path from 'node:path';
|
|
|
61
61
|
import { spawnSync } from 'node:child_process';
|
|
62
62
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
63
63
|
import { COVERAGE_ASSET, COVERAGE_RECEIPT_ASSET, bindCoverageToReceipt, verifyCoverageSidecar } from './corpus-coverage-sidecar.mjs';
|
|
64
|
+
import { retryTransientResult } from './transient-gh.mjs';
|
|
64
65
|
|
|
65
66
|
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
66
67
|
const HEX64 = /^[0-9a-f]{64}$/;
|
|
@@ -76,7 +77,10 @@ const REQUIRED_ASSETS = [ARCHIVE_ASSET, SIGNATURE_ASSET, RECEIPT_ASSET, RECALL_A
|
|
|
76
77
|
/** ADR-0091 D4: judge at most this many of the newest generations, then use the committed bootstrap. */
|
|
77
78
|
export const SEARCH_BOUND = 5;
|
|
78
79
|
|
|
79
|
-
|
|
80
|
+
// Every `gh` call below is a read (release list, release view, small-asset download). A transient failure is
|
|
81
|
+
// retried, bounded, before it is recorded as `indeterminate` (scripts/transient-gh.mjs): an unretried blip here
|
|
82
|
+
// used to send the whole night back to the committed bootstrap seed -- a full rebuild dressed as a quiet night.
|
|
83
|
+
const defaultRun = (command, args, options) => retryTransientResult(() => spawnSync(command, args, { encoding: 'utf8', ...options }));
|
|
80
84
|
|
|
81
85
|
function ghJson(run, args) {
|
|
82
86
|
const result = run('gh', args, { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] });
|
|
@@ -25,6 +25,7 @@ import { storeRoot } from '../kb/store-root.mjs';
|
|
|
25
25
|
import { captureGistSources } from './gist-receipts.mjs';
|
|
26
26
|
import { projectSourceStore, RUNTIME_LEDGER_KIND } from './rvf-generation.mjs';
|
|
27
27
|
import { compareKnowledgeInputs, fromCoverage as knowledgeFromCoverage, fromSeed as knowledgeFromSeed } from './knowledge-input-digest.mjs';
|
|
28
|
+
import { defaultBaseDelayMs } from './transient-gh.mjs';
|
|
28
29
|
|
|
29
30
|
export { rebuildCorpusAggregates };
|
|
30
31
|
|
|
@@ -352,8 +353,10 @@ export async function acquireSealedGeneration({ maxAttempts = 3, assetsDir = nul
|
|
|
352
353
|
const settled = await build(observation, storeOutcomes);
|
|
353
354
|
const remaining = planReconciliation({ coverage: settled, ledger: currentLedger(), assetsDir })
|
|
354
355
|
.filter((item) => !recorded(item.store));
|
|
356
|
+
// A repository row the shipped validator accepts WITH a carry -- a transiently unreadable source pinned at its
|
|
357
|
+
// seed commit (source-coverage.mjs) -- is resolved for this generation, exactly like a recorded store outcome.
|
|
355
358
|
const unresolved = settled.rows.filter((row) => row.disposition === 'eligible' && row.status !== 'CURRENT'
|
|
356
|
-
&& !(row.kind === 'repository' && recorded(row.artifact?.store)));
|
|
359
|
+
&& !(row.kind === 'repository' && (recorded(row.artifact?.store) || (row.carry && eligibleRepositoryStanding(row) === 'shipped'))));
|
|
357
360
|
attempts.push({ attempt, plan, ...reconciliation, ...pruning, ...aggregates,
|
|
358
361
|
remainingArtifacts: remaining.length, unresolvedSources: unresolved.length });
|
|
359
362
|
if (!remaining.length && !unresolved.length) {
|
|
@@ -635,11 +638,13 @@ export function dispositionForFailedStore({ assetsDir, ledger, item, attempts, r
|
|
|
635
638
|
|
|
636
639
|
// Only a TRANSIENT failure is retried, exactly once, in a fresh directory: `<store>-retry1`. The first
|
|
637
640
|
// attempt's directory is never reused -- a clone or a half-written worker output from the failed
|
|
638
|
-
// attempt must not be mistaken for the retry's own (the pre-D5 path collided).
|
|
641
|
+
// attempt must not be mistaken for the retry's own (the pre-D5 path collided). The retry waits
|
|
642
|
+
// TRANSIENT_RETRY_DELAY_MS first: an immediate re-clone into the same reset connection learns nothing.
|
|
639
643
|
export function workerRootFor(workspace, store, retry) {
|
|
640
644
|
return path.join(workspace, 'workers', retry === 0 ? store : `${store}-retry${retry}`);
|
|
641
645
|
}
|
|
642
646
|
export const MAX_TRANSIENT_RETRIES = 1;
|
|
647
|
+
export const TRANSIENT_RETRY_DELAY_MS = 30_000;
|
|
643
648
|
|
|
644
649
|
export async function executeReconciliation({
|
|
645
650
|
plan,
|
|
@@ -651,6 +656,8 @@ export async function executeReconciliation({
|
|
|
651
656
|
signal,
|
|
652
657
|
priorCoverage = null,
|
|
653
658
|
log = (line) => console.log(line),
|
|
659
|
+
retryDelayMs = defaultBaseDelayMs(TRANSIENT_RETRY_DELAY_MS),
|
|
660
|
+
sleep = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); }),
|
|
654
661
|
}) {
|
|
655
662
|
if (!Array.isArray(plan) || !Number.isSafeInteger(concurrency) || concurrency < 1 || concurrency > 10) {
|
|
656
663
|
fail('reconciliation plan or worker concurrency is invalid');
|
|
@@ -750,8 +757,10 @@ export async function executeReconciliation({
|
|
|
750
757
|
lastError = error;
|
|
751
758
|
const retrying = isRetryable(error) && retry < MAX_TRANSIENT_RETRIES;
|
|
752
759
|
log(`[corpus-reconcile] ${item.store}: attempt ${attempts} failed -- ${error.message}`
|
|
753
|
-
+ (retrying ?
|
|
760
|
+
+ (retrying ? `; retrying once in a fresh worker directory after ${retryDelayMs / 1000}s` : '; not retried'));
|
|
754
761
|
if (!retrying) break;
|
|
762
|
+
await sleep(retryDelayMs);
|
|
763
|
+
if (controller.signal.aborted) throw abortError(controller.signal);
|
|
755
764
|
}
|
|
756
765
|
}
|
|
757
766
|
return { ok: false, attempts, error: lastError };
|
|
@@ -871,7 +880,33 @@ async function observeSourceOnly({ owner, assetsDir }) {
|
|
|
871
880
|
const externalFile = path.join(assets, 'external-sources.json');
|
|
872
881
|
const policy = fs.existsSync(externalFile) ? readJson(externalFile, 'external source policy') : { sources: [] };
|
|
873
882
|
if (!Array.isArray(policy.sources)) fail('external source policy has no sources array');
|
|
874
|
-
|
|
883
|
+
const ledgerFile = path.join(assets, 'RVF-GENERATIONS.json');
|
|
884
|
+
const ledger = fs.existsSync(ledgerFile) ? readJson(ledgerFile, 'RVF generation ledger') : { stores: {} };
|
|
885
|
+
return observeSourceUniverse({ owner, externalSources: policy.sources,
|
|
886
|
+
carriedCommits: transientCarryCommits({ ledger, priorCoverage: readPriorCoverage(assets) }) });
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
/**
|
|
890
|
+
* The seed's own commits, per store: what an external source unreadable tonight is pinned at (one night, STALE
|
|
891
|
+
* with a carry) instead of failing the whole observation. Read from the seed ledger, never invented -- and a store
|
|
892
|
+
* the SEED's sealed coverage already shows carried this way is left out, so the second consecutive night fails
|
|
893
|
+
* for that source exactly as before rather than carrying it forever.
|
|
894
|
+
*/
|
|
895
|
+
export function transientCarryCommits({ ledger, priorCoverage = null }) {
|
|
896
|
+
// No sealed coverage to read history from (the bootstrap lineage, or an invalid sidecar) = no carry at all:
|
|
897
|
+
// the allowance exists only where the previous night can prove it was not already spent.
|
|
898
|
+
if (!Array.isArray(priorCoverage?.rows)) return {};
|
|
899
|
+
// ANY carried seed row spends it -- a failed refresh last night and an unreadable upstream tonight must not
|
|
900
|
+
// alternate forever either.
|
|
901
|
+
const alreadyCarried = new Set(priorCoverage.rows
|
|
902
|
+
.filter((row) => row?.kind === 'repository' && row.carry)
|
|
903
|
+
.map((row) => String(row.artifact?.store || '').toLowerCase()));
|
|
904
|
+
return Object.fromEntries(Object.entries(ledger?.stores || {})
|
|
905
|
+
.filter(([store, row]) => HEX40.test(String(row?.sourceCommit || '').toLowerCase()) && !alreadyCarried.has(store.toLowerCase()))
|
|
906
|
+
.map(([store, row]) => {
|
|
907
|
+
const sourceCommit = String(row.sourceCommit).toLowerCase();
|
|
908
|
+
return [store.toLowerCase(), { sourceCommit, committedAt: carriedCommittedAt({ priorCoverage, store, sourceCommit }) }];
|
|
909
|
+
}));
|
|
875
910
|
}
|
|
876
911
|
|
|
877
912
|
export async function acquireCorpusGeneration({ owner = 'ruvnet', assetsDir, workspaceDir,
|
|
@@ -22,6 +22,10 @@
|
|
|
22
22
|
// WARNING a superseded or degraded night, a stand-down tonight, an unknown outcome, a long run
|
|
23
23
|
// GREEN none of the above
|
|
24
24
|
//
|
|
25
|
+
// THE FIRST LINE IS THE NIGHT'S VERDICT (2026-10-20, summarizeNight): what the night did -- published,
|
|
26
|
+
// no-change, degraded with the stores named, failed with the stage named, superseded, canary-rejected, still
|
|
27
|
+
// running, stood down -- in Eastern time, result first, so the owner never learns of a break from a page.
|
|
28
|
+
//
|
|
25
29
|
// Exit code is the contract (same as scripts/github-health-watch.mjs): 0 = GREEN or WARNING,
|
|
26
30
|
// 1 = RED or the evidence could not be gathered. corpus-watchdog.yml runs on a genuine `schedule:`,
|
|
27
31
|
// so its red conclusion fires ntfy-alerts.yml's workflow_run listener — no secret needed here.
|
|
@@ -43,6 +47,7 @@ import { CODE_TAG_PATTERN, isCorpusReleaseTag } from './release-channel-kind.mjs
|
|
|
43
47
|
import { AGGREGATE_ASSET, SIGNING_PUBLIC_KEY_FILE } from './approved-runtime.mjs';
|
|
44
48
|
import { COVERAGE_ASSET, COVERAGE_RECEIPT_ASSET, verifyCoverageSidecar } from './corpus-coverage-sidecar.mjs';
|
|
45
49
|
import { parseCorpusGeneration } from './corpus-promotion.mjs';
|
|
50
|
+
import { retryTransientResult } from './transient-gh.mjs';
|
|
46
51
|
|
|
47
52
|
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
48
53
|
export const RED = 'RED';
|
|
@@ -255,7 +260,9 @@ export function judgeCorpusHealth(input, now) {
|
|
|
255
260
|
// ── Gathering (read-only gh). Everything above is pure; everything below talks to GitHub. ──────────
|
|
256
261
|
|
|
257
262
|
function defaultGh(args) {
|
|
258
|
-
|
|
263
|
+
// A transient gh failure is retried, bounded (scripts/transient-gh.mjs); the judge must not page RED on weather.
|
|
264
|
+
const result = retryTransientResult(() => spawnSync(process.env.RUVNET_GH_COMMAND || 'gh', args,
|
|
265
|
+
{ encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: 5 * 60_000 }));
|
|
259
266
|
if (result.error || result.status !== 0) {
|
|
260
267
|
throw new Error(`gh ${args.slice(0, 3).join(' ')} failed: ${String(result.error?.message || result.stderr || `exit ${result.status}`).trim().slice(0, 300)}`);
|
|
261
268
|
}
|
|
@@ -320,17 +327,25 @@ function corpusRunEvidence({ gh, repo, run, scratch }) {
|
|
|
320
327
|
}
|
|
321
328
|
}
|
|
322
329
|
|
|
330
|
+
/**
|
|
331
|
+
* The protected-release corpus runs created since `since` (ms), each with its outcome evidence. Shared with the
|
|
332
|
+
* dispatcher's retry slot (scripts/corpus-dispatch-decision.mjs --retry-slot), so both read one definition of
|
|
333
|
+
* "tonight's runs" and one classifier.
|
|
334
|
+
*/
|
|
335
|
+
export function gatherCorpusRuns({ repo, since, gh = defaultGh, scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'corpus-watchdog-')) }) {
|
|
336
|
+
return JSON.parse(gh(['run', 'list', '--repo', repo, '--workflow', 'protected-release.yml', '--limit', '60',
|
|
337
|
+
'--json', 'databaseId,attempt,displayTitle,status,conclusion,createdAt,updatedAt']))
|
|
338
|
+
.filter((run) => CORPUS_RUN_TITLE.test(String(run.displayTitle || '')) && (ms(run.createdAt) ?? 0) >= since)
|
|
339
|
+
.map((run) => ({ id: run.databaseId, title: run.displayTitle, status: run.status, conclusion: run.conclusion,
|
|
340
|
+
createdAt: run.createdAt, updatedAt: run.updatedAt, ...corpusRunEvidence({ gh, repo, run, scratch }) }));
|
|
341
|
+
}
|
|
342
|
+
|
|
323
343
|
export async function gatherCorpusHealthInput({ repo, now, gh = defaultGh, root = ROOT, scratch = fs.mkdtempSync(path.join(os.tmpdir(), 'corpus-watchdog-')) }) {
|
|
324
344
|
const at = now.getTime();
|
|
325
|
-
const recent = (iso) => (ms(iso) ?? 0) >= at - 3 * 24 * HOUR;
|
|
326
345
|
const dispatcherRuns = JSON.parse(gh(['run', 'list', '--repo', repo, '--workflow', 'corpus-nightly-dispatch.yml', '--limit', '30',
|
|
327
346
|
'--json', 'databaseId,attempt,event,status,conclusion,createdAt,updatedAt']))
|
|
328
347
|
.map((run) => ({ id: run.databaseId, attempt: run.attempt, event: run.event, status: run.status, conclusion: run.conclusion, createdAt: run.createdAt }));
|
|
329
|
-
const corpusRuns =
|
|
330
|
-
'--json', 'databaseId,attempt,displayTitle,status,conclusion,createdAt,updatedAt']))
|
|
331
|
-
.filter((run) => CORPUS_RUN_TITLE.test(String(run.displayTitle || '')) && recent(run.createdAt))
|
|
332
|
-
.map((run) => ({ id: run.databaseId, title: run.displayTitle, status: run.status, conclusion: run.conclusion,
|
|
333
|
-
createdAt: run.createdAt, updatedAt: run.updatedAt, ...corpusRunEvidence({ gh, repo, run, scratch }) }));
|
|
348
|
+
const corpusRuns = gatherCorpusRuns({ repo, since: at - 3 * 24 * HOUR, gh, scratch });
|
|
334
349
|
const releases = JSON.parse(gh(['release', 'list', '--repo', repo, '--limit', '100', '--json', 'tagName,publishedAt,isDraft,isPrerelease']));
|
|
335
350
|
// PROMOTED only: a staged candidate (and a bootstrap seed) is a prerelease no customer receives.
|
|
336
351
|
const corpusReleases = releases.filter((row) => !row.isDraft && !row.isPrerelease && isCorpusReleaseTag(row.tagName))
|
|
@@ -364,8 +379,72 @@ export async function gatherCorpusHealthInput({ repo, now, gh = defaultGh, root
|
|
|
364
379
|
return { corpusReleases, corpusRuns, dispatcherRuns, codeRelease, deferral, latestRelease };
|
|
365
380
|
}
|
|
366
381
|
|
|
382
|
+
/** A time the owner reads: local Eastern, 12-hour, named zone -- never UTC or "Z". */
|
|
383
|
+
export function formatEastern(when) {
|
|
384
|
+
const t = when instanceof Date ? when.getTime() : ms(when);
|
|
385
|
+
if (t === null || !Number.isFinite(t)) return 'an unknown time';
|
|
386
|
+
const parts = Object.fromEntries(new Intl.DateTimeFormat('en-US', { timeZone: 'America/New_York', month: 'short', day: 'numeric',
|
|
387
|
+
hour: 'numeric', minute: '2-digit', hour12: true }).formatToParts(new Date(t)).map((part) => [part.type, part.value]));
|
|
388
|
+
return `${parts.hour}:${parts.minute} ${parts.dayPeriod.toLowerCase().replace(/^(a|p)m$/, '$1.m.')} ET ${parts.month} ${parts.day}`;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** The job the night died in, named as the owner knows the stage, from the terminal outcome record or the job list. */
|
|
392
|
+
const STAGE_NAMES = { identity: 'authorize (corpus-identity)', prepare: 'prepare/seal (seal-exact-corpus-candidate)',
|
|
393
|
+
authorize: 'authenticate (corpus-authorize)', publish: 'publisher (corpus-publish)', canary: 'customer canary', promote: 'promote' };
|
|
394
|
+
const JOB_NAMES = { 'authorize-corpus-generation': 'identity', 'seal-exact-corpus-candidate': 'prepare', 'authenticate-corpus-preparation-provenance': 'authorize',
|
|
395
|
+
'protected-corpus-publisher': 'publish', 'customer-canary': 'canary', 'promote-canaried-corpus': 'promote' };
|
|
396
|
+
export function failedStage(run) {
|
|
397
|
+
const recorded = Object.entries(run?.outcomeRecord?.jobs || {}).find(([, result]) => result === 'failure')?.[0];
|
|
398
|
+
const listed = (run?.jobs || []).find((job) => job.conclusion === 'failure' && JOB_NAMES[job.name]);
|
|
399
|
+
const stage = recorded || (listed ? JOB_NAMES[listed.name] : null);
|
|
400
|
+
return stage ? STAGE_NAMES[stage] || stage : `an unnamed job (conclusion ${run?.conclusion || 'unknown'})`;
|
|
401
|
+
}
|
|
402
|
+
|
|
403
|
+
/**
|
|
404
|
+
* THE ONE LINE the owner reads first: what the night did, result first, in Eastern time. `input` is
|
|
405
|
+
* gatherCorpusHealthInput's; the night is the newest corpus run of the last 24h and every retry of it.
|
|
406
|
+
* Never learned from a page: the watchdog writes this before ntfy ever fires.
|
|
407
|
+
*/
|
|
408
|
+
export function summarizeNight(input, now) {
|
|
409
|
+
const at = now instanceof Date ? now.getTime() : Number(now);
|
|
410
|
+
const inWindow = (iso) => { const t = ms(iso); return t !== null && t <= at && at - t <= TONIGHT_WINDOW_MS; };
|
|
411
|
+
const runs = (input?.corpusRuns || []).filter((run) => inWindow(run.createdAt))
|
|
412
|
+
.map((run) => ({ ...run, outcome: run.outcome ?? classifyCorpusRun(run) })).sort((a, b) => ms(b.createdAt) - ms(a.createdAt));
|
|
413
|
+
const dispatcher = (input?.dispatcherRuns || []).filter((d) => inWindow(d.createdAt)).sort((a, b) => ms(b.createdAt) - ms(a.createdAt))[0];
|
|
414
|
+
const latest = input?.latestRelease;
|
|
415
|
+
const attempts = runs.length > 1 ? ` (attempt ${runs.length} of the night; ${runs.length - 1} retried)` : '';
|
|
416
|
+
const keep = latest ? `customers keep ${latest.tag} (${hours(Math.max(0, at - (ms(latest.generation || latest.publishedAt) ?? at)))} old)` : 'no promoted generation is observed';
|
|
417
|
+
if (!runs.length) {
|
|
418
|
+
if (!dispatcher) return { outcome: 'no-run', line: `night: no corpus run in the last 24h (no dispatcher run either); ${keep}` };
|
|
419
|
+
if (dispatcher.status !== 'completed') return { outcome: 'dispatching', line: `night: dispatcher run ${dispatcher.id} is ${dispatcher.status} (started ${formatEastern(dispatcher.createdAt)})` };
|
|
420
|
+
if (dispatcher.conclusion !== 'success') return { outcome: 'dispatcher-failed', line: `night: dispatcher run ${dispatcher.id} concluded ${dispatcher.conclusion} at ${formatEastern(dispatcher.createdAt)}, nothing was dispatched; ${keep}` };
|
|
421
|
+
return { outcome: 'stood-down', line: `night: stood down at ${formatEastern(dispatcher.createdAt)} (CORPUS_NIGHTLY off, or no install-verified runtime); ${keep}` };
|
|
422
|
+
}
|
|
423
|
+
const [run] = runs;
|
|
424
|
+
const when = formatEastern(run.updatedAt || run.createdAt);
|
|
425
|
+
const degraded = input?.deferral?.observable && input.deferral.degraded && latest && input.deferral.newestTag === latest.tag
|
|
426
|
+
&& (input.deferral.degraded.carried.length || input.deferral.degraded.missing.length) ? input.deferral.degraded : null;
|
|
427
|
+
switch (run.outcome) {
|
|
428
|
+
case 'published': {
|
|
429
|
+
const tag = latest && (ms(latest.publishedAt) ?? 0) >= (ms(run.createdAt) ?? 0) ? latest.tag : 'a new generation';
|
|
430
|
+
if (degraded) {
|
|
431
|
+
return { outcome: 'degraded', line: `night: degraded -- published ${tag} at ${when}${attempts} with ${degraded.carried.length} carried `
|
|
432
|
+
+ `(${degraded.carried.join(', ') || 'none'}) and ${degraded.missing.length} missing (${degraded.missing.join(', ') || 'none'}), each named in CORPUS-COVERAGE.json` };
|
|
433
|
+
}
|
|
434
|
+
return { outcome: 'published', line: `night: published ${tag} at ${when}${attempts}` };
|
|
435
|
+
}
|
|
436
|
+
case 'no-change': return { outcome: 'no-change', line: `night: no-change at ${when}${attempts} -- every knowledge input equals the promoted generation; ${keep}` };
|
|
437
|
+
case 'superseded': return { outcome: 'superseded', line: `night: superseded at ${when} -- a newer code release landed; the next night builds at it; ${keep}` };
|
|
438
|
+
case 'canary-rejected': return { outcome: 'canary-rejected', line: `night: canary-rejected at ${when}${attempts} -- a clean customer install could not apply the candidate (GLOBAL: not retried); ${keep}` };
|
|
439
|
+
case 'in-progress': return { outcome: 'in-progress', line: `night: run ${run.id} still in progress, started ${formatEastern(run.createdAt)}${attempts}` };
|
|
440
|
+
case 'failed': return { outcome: 'failed', line: `night: failed in ${failedStage(run)} at ${when}${attempts}; ${keep}` };
|
|
441
|
+
default: return { outcome: 'unknown', line: `night: run ${run.id} succeeded at ${when} but reached no outcome${attempts}; ${keep}` };
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
367
445
|
export function renderReport(result, now) {
|
|
368
|
-
const lines = [`corpus-watchdog ${result.verdict}
|
|
446
|
+
const lines = [`corpus-watchdog ${result.verdict} -- ${result.night?.line || `judged ${formatEastern(now)}`}`,
|
|
447
|
+
` judged ${formatEastern(now)} (${now.toISOString()})`];
|
|
369
448
|
for (const f of result.findings) lines.push(` [${f.level}] ${f.check}: ${f.detail}`);
|
|
370
449
|
return `${lines.join('\n')}\n`;
|
|
371
450
|
}
|
|
@@ -380,7 +459,8 @@ async function main(argv = process.argv.slice(2)) {
|
|
|
380
459
|
}
|
|
381
460
|
let result;
|
|
382
461
|
try {
|
|
383
|
-
|
|
462
|
+
const input = await gatherCorpusHealthInput({ repo, now });
|
|
463
|
+
result = { ...judgeCorpusHealth(input, now), night: summarizeNight(input, now) };
|
|
384
464
|
} catch (error) {
|
|
385
465
|
// Could not observe = could not prove health. Absence of evidence is failure.
|
|
386
466
|
process.stderr.write(`corpus-watchdog RED: evidence could not be gathered: ${error.message}\n`);
|
|
@@ -34,6 +34,7 @@ import { canonicalJson, digest, validateGistAggregateReceipt } from './coverage-
|
|
|
34
34
|
import { promoteArtifactSet } from '../kb/incremental-refresh.mjs';
|
|
35
35
|
import { writeRvfGeneration } from './rvf-generation.mjs';
|
|
36
36
|
import { fetchGistGitSnapshot } from './gist-git-transport.mjs';
|
|
37
|
+
import { isTransientText, retryTransientAsync } from './transient-gh.mjs';
|
|
37
38
|
|
|
38
39
|
const DEFAULT_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
39
40
|
const TEXT_EXT = new Set(['.md', '.markdown', '.txt', '.rst']);
|
|
@@ -402,9 +403,15 @@ function reusableCachedGist(cached, stub) {
|
|
|
402
403
|
return 'reuse';
|
|
403
404
|
}
|
|
404
405
|
|
|
406
|
+
// The Git snapshot is one `ls-remote` + one bare clone per gist against gist.github.com. A transient transport
|
|
407
|
+
// failure (reset, timeout, "could not read Username", or the gist moving mid-capture) is retried, bounded, so
|
|
408
|
+
// one blip on one of ~180 gists cannot fail the aggregate and with it the night. A gist that is gone
|
|
409
|
+
// ("repository not found") is an answer and is thrown at once.
|
|
410
|
+
const transientGistSnapshot = (error) => isTransientText(error?.message) || /moved while Git snapshot was captured/.test(String(error?.message || ''));
|
|
411
|
+
|
|
405
412
|
export async function captureGistSources({ observation, cache = null, fetchDetail = null,
|
|
406
413
|
fetchRaw = defaultFetchRaw, fetchGitSnapshot = fetchGistGitSnapshot, signal,
|
|
407
|
-
now = () => new Date().toISOString() } = {}) {
|
|
414
|
+
now = () => new Date().toISOString(), snapshotRetryDelayMs = 15_000 } = {}) {
|
|
408
415
|
const stubs = observation?.gists?.rows;
|
|
409
416
|
if (!Array.isArray(stubs) || stubs.some(({ id }) => !HEX_GIST.test(String(id || '')))
|
|
410
417
|
|| new Set(stubs.map(({ id }) => id)).size !== stubs.length) {
|
|
@@ -420,7 +427,8 @@ export async function captureGistSources({ observation, cache = null, fetchDetai
|
|
|
420
427
|
// The production/no-key route never spends one REST detail request per gist and never trusts
|
|
421
428
|
// a body-free cache or timestamps as content identity. A full bare clone gives one immutable
|
|
422
429
|
// Git commit/tree; every observed raw_url's blob identity must match its HEAD file bytes.
|
|
423
|
-
const snapshot = await fetchGitSnapshot({ owner: observation.owner, stub, signal })
|
|
430
|
+
const snapshot = await retryTransientAsync(() => fetchGitSnapshot({ owner: observation.owner, stub, signal }),
|
|
431
|
+
{ isTransient: transientGistSnapshot, baseDelayMs: snapshotRetryDelayMs });
|
|
424
432
|
const files = [];
|
|
425
433
|
for (const source of snapshot.files) {
|
|
426
434
|
const filename = source.filename;
|
package/scripts/release.mjs
CHANGED
|
@@ -71,6 +71,7 @@ import {
|
|
|
71
71
|
import { bindCoverageToReceipt, writeCoverageAssets } from './corpus-coverage-sidecar.mjs';
|
|
72
72
|
import { describeDegraded } from './corpus-store-failure.mjs';
|
|
73
73
|
import { assertNoNewerCorpusGeneration } from './code-release-corpus.mjs';
|
|
74
|
+
import { retryingGh } from './transient-gh.mjs';
|
|
74
75
|
|
|
75
76
|
const ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
|
|
76
77
|
const PUBLISH = process.argv.includes('--publish');
|
|
@@ -141,11 +142,17 @@ const compareCodeTags = (left, right) => {
|
|
|
141
142
|
return Math.sign(a[0] - b[0] || a[1] - b[1] || a[2] - b[2]);
|
|
142
143
|
};
|
|
143
144
|
|
|
144
|
-
/**
|
|
145
|
+
/**
|
|
146
|
+
* The one gh seam both corpus entry points use (RUVNET_GH_COMMAND / RUVNET_GH_SCRIPT for tests). Reads and the
|
|
147
|
+
* idempotent `release edit` retry a transient failure, bounded (scripts/transient-gh.mjs): on 2026-09-29 the
|
|
148
|
+
* night was red on "cannot confirm the promoted corpus release" -- one `release view` that failed right after
|
|
149
|
+
* the write had succeeded. `release create` runs exactly once; the absence proof above it is what makes a
|
|
150
|
+
* repeat safe to refuse.
|
|
151
|
+
*/
|
|
145
152
|
function corpusGh(env, run) {
|
|
146
|
-
const
|
|
147
|
-
|
|
148
|
-
|
|
153
|
+
const gh = retryingGh({ run, command: env.RUVNET_GH_COMMAND || 'gh', prefix: env.RUVNET_GH_SCRIPT ? [env.RUVNET_GH_SCRIPT] : [],
|
|
154
|
+
options: { encoding: 'utf8', stdio: ['ignore', 'pipe', 'pipe'] },
|
|
155
|
+
onRetry: ({ attempt, args, delayMs }) => console.error(`[corpus-seed] gh ${args.slice(0, 2).join(' ')} failed transiently (attempt ${attempt}); retrying in ${delayMs / 1000}s`) });
|
|
149
156
|
const ghJson = (args, label) => {
|
|
150
157
|
const result = gh(args);
|
|
151
158
|
if (result.error || result.status !== 0) {
|
|
@@ -9,6 +9,7 @@ import { readRvfGenerations, sha256File } from './rvf-generation.mjs';
|
|
|
9
9
|
import { canonicalJson, coverageGenerationFor, digest, validateGistAggregateReceipt } from './coverage-integrity.mjs';
|
|
10
10
|
import { repositoryNames } from '../kb/card-lane.mjs';
|
|
11
11
|
import { rootNeverMaterialized, storeRoot } from '../kb/store-root.mjs';
|
|
12
|
+
import { isTransientText, retryTransient, sleepSync } from './transient-gh.mjs';
|
|
12
13
|
|
|
13
14
|
export { canonicalJson, digest } from './coverage-integrity.mjs';
|
|
14
15
|
|
|
@@ -52,6 +53,11 @@ function runGh(args) {
|
|
|
52
53
|
return result.stdout;
|
|
53
54
|
}
|
|
54
55
|
|
|
56
|
+
// Every observation read below goes through this: a transient failure (rate limit, 5xx, a reset) is retried,
|
|
57
|
+
// bounded, before it can fail the night (scripts/transient-gh.mjs). Applied around the INJECTED `gh`, not
|
|
58
|
+
// inside runGh, so the fault-injection suite exercises the same retry the nightly runs.
|
|
59
|
+
const retryingRead = (gh, sleep) => (args) => retryTransient(() => gh(args), { sleep });
|
|
60
|
+
|
|
55
61
|
// A fork's disposition depends on how many commits it carries that its upstream does not. The
|
|
56
62
|
// compare is pinned to the two heads THIS observation recorded (never to branch names, which move),
|
|
57
63
|
// and is run against the upstream repository in GitHub's documented cross-fork form
|
|
@@ -80,7 +86,8 @@ function observeForkDelta(owner, node, gh) {
|
|
|
80
86
|
};
|
|
81
87
|
}
|
|
82
88
|
|
|
83
|
-
export function observeRepositories(owner, { gh = runGh } = {}) {
|
|
89
|
+
export function observeRepositories(owner, { gh: rawGh = runGh, sleep = sleepSync } = {}) {
|
|
90
|
+
const gh = retryingRead(rawGh, sleep);
|
|
84
91
|
const rows = [];
|
|
85
92
|
const pages = [];
|
|
86
93
|
let cursor = '';
|
|
@@ -103,10 +110,11 @@ export function observeRepositories(owner, { gh = runGh } = {}) {
|
|
|
103
110
|
return { rows: withDeltas, expected, pages };
|
|
104
111
|
}
|
|
105
112
|
|
|
106
|
-
// The row an external source reports
|
|
107
|
-
// stored content classifies CURRENT, nothing is re-ingested, and the row says plainly
|
|
108
|
-
|
|
109
|
-
|
|
113
|
+
// The row an external source reports when its upstream cannot be read: exactly what was last observed (or last
|
|
114
|
+
// ingested), so the stored content classifies CURRENT, nothing is re-ingested, and the row says plainly why.
|
|
115
|
+
// `gone` is the policy's ifGone entry (404/410); for a source unreadable for a moment it is the seed ledger's
|
|
116
|
+
// own commit, with `reason` naming the failure.
|
|
117
|
+
function unavailableExternalRow(source, gone, reason) {
|
|
110
118
|
if (!/^[0-9a-f]{40}$/i.test(String(gone.sourceCommit || ''))) {
|
|
111
119
|
throw new Error(`external source ${source.repository}: ifGone.sourceCommit must be the 40-hex commit last ingested`);
|
|
112
120
|
}
|
|
@@ -114,12 +122,27 @@ function goneExternalRow(source) {
|
|
|
114
122
|
name: source.repository.split('/')[1], fullName: source.repository, storeName: source.store,
|
|
115
123
|
url: `https://github.com/${source.repository}`, description: gone.description ?? null, homepageUrl: null,
|
|
116
124
|
isFork: false, upstreamIsFork: false, isArchived: false, isDisabled: false,
|
|
117
|
-
updatedAt: gone.updatedAt ?? null, pushedAt: gone.pushedAt ?? null, upstreamUnavailable: true,
|
|
125
|
+
updatedAt: gone.updatedAt ?? null, pushedAt: gone.pushedAt ?? null, upstreamUnavailable: true, upstreamUnavailableReason: reason,
|
|
118
126
|
defaultBranchRef: { name: gone.defaultBranch || 'HEAD', target: { oid: gone.sourceCommit, committedDate: gone.committedAt ?? null } },
|
|
119
127
|
};
|
|
120
128
|
}
|
|
121
129
|
|
|
122
|
-
|
|
130
|
+
/** The one reason a transient carry records; scripts/corpus-reconcile.mjs refuses a second consecutive one by it. */
|
|
131
|
+
export const TRANSIENT_UNREADABLE_REASON = 'was unreadable this observation (transient failure after the bounded retry); it is read again next night';
|
|
132
|
+
/** The carry record a transiently unreadable source ships: STALE, not CURRENT, and counted toward the degraded bound. */
|
|
133
|
+
export const TRANSIENT_CARRY_REASON = 'transient: upstream unreadable';
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* `carriedCommits`: { <store>: { sourceCommit, committedAt } } from the seed's own generation ledger, already stripped
|
|
137
|
+
* of any store the seed's coverage shows was ALREADY carried this way (corpus-reconcile.mjs transientCarryCommits),
|
|
138
|
+
* so the same source is never carried two nights running. A source whose read still fails TRANSIENTLY after the
|
|
139
|
+
* bounded retry (rate limit, 5xx, a reset; measured 2026-09-26) is pinned at that commit for ONE night: its row
|
|
140
|
+
* classifies STALE with a `carry` record (nobody read upstream tonight, so it is not called current), it counts
|
|
141
|
+
* toward max(3, 5% of eligible) with every other carried store, and the next night reads it again. Without a prior
|
|
142
|
+
* commit there is nothing honest to carry, and the observation fails.
|
|
143
|
+
*/
|
|
144
|
+
export function observeExternalRepositories(sources, { gh: rawGh = runGh, sleep = sleepSync, carriedCommits = null } = {}) {
|
|
145
|
+
const gh = retryingRead(rawGh, sleep);
|
|
123
146
|
const rows = [];
|
|
124
147
|
const pages = [];
|
|
125
148
|
for (const source of sources) {
|
|
@@ -134,13 +157,28 @@ export function observeExternalRepositories(sources, { gh = runGh } = {}) {
|
|
|
134
157
|
? JSON.parse(gh(['api', `repos/${source.repository}/commits/${repo.default_branch}`]))
|
|
135
158
|
: null;
|
|
136
159
|
} catch (error) {
|
|
137
|
-
// An upstream that GitHub reports gone (404/410) must not stop every night for every customer
|
|
138
|
-
//
|
|
139
|
-
//
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
160
|
+
// An upstream that GitHub reports gone (404/410) must not stop every night for every customer: a source
|
|
161
|
+
// whose policy entry records `ifGone` (the commit it was last ingested at) is carried there. A source that
|
|
162
|
+
// is merely unreadable tonight (still transient after the retry) is carried at the seed's own commit when
|
|
163
|
+
// it has one. Anything else -- a bare 403, a mistyped repository, no prior bytes -- still fails the observation.
|
|
164
|
+
const message = String(error.message);
|
|
165
|
+
const gone = source.ifGone && /\(HTTP (404|410)\)/.test(message);
|
|
166
|
+
const prior = carriedCommits?.[String(source.store).toLowerCase()];
|
|
167
|
+
const carried = String(prior?.sourceCommit ?? prior ?? '').toLowerCase();
|
|
168
|
+
const unreadable = !gone && isTransientText(message) && /^[0-9a-f]{40}$/.test(carried);
|
|
169
|
+
if (!gone && !unreadable) throw error;
|
|
170
|
+
// FIXED shapes, never the error text or a timestamp: the observation digest feeds the no-change decision,
|
|
171
|
+
// so a night that carried one unreadable source and saw nothing else move must still read as unchanged.
|
|
172
|
+
const kind = gone ? 'gone' : 'transient';
|
|
173
|
+
const reason = gone ? 'is no longer reachable (HTTP 404/410)' : TRANSIENT_UNREADABLE_REASON;
|
|
174
|
+
// committedAt comes from the seed's own coverage row (corpus-reconcile.mjs carriedCommittedAt), so the customer's
|
|
175
|
+
// "oldest carried" line dates the pinned commit instead of showing an unknown date.
|
|
176
|
+
const at = gone ? source.ifGone : { sourceCommit: carried, committedAt: prior?.committedAt ?? null };
|
|
177
|
+
process.stderr.write(`[source-coverage] external source ${source.repository} ${reason}; carrying the last ingested content (${at.sourceCommit})`
|
|
178
|
+
+ `${gone ? '' : ` -- ${message.replace(/\s+/g, ' ').trim().slice(0, 200)}`}\n`);
|
|
179
|
+
rows.push({ ...unavailableExternalRow(source, at, reason), upstreamUnavailableKind: kind });
|
|
180
|
+
pages.push({ repository: source.repository, responseDigest: digest(gone ? { gone: source.repository, ifGone: source.ifGone }
|
|
181
|
+
: { unavailable: source.repository, kind, at: carried }), count: 1, terminal: true });
|
|
144
182
|
continue;
|
|
145
183
|
}
|
|
146
184
|
const row = {
|
|
@@ -164,9 +202,11 @@ export function observeSourceUniverse({
|
|
|
164
202
|
externalSources = [],
|
|
165
203
|
gh = runGh,
|
|
166
204
|
observedAt = new Date().toISOString(),
|
|
205
|
+
sleep = sleepSync,
|
|
206
|
+
carriedCommits = null,
|
|
167
207
|
} = {}) {
|
|
168
|
-
const primary = observeRepositories(owner, { gh });
|
|
169
|
-
const external = observeExternalRepositories(externalSources, { gh });
|
|
208
|
+
const primary = observeRepositories(owner, { gh, sleep });
|
|
209
|
+
const external = observeExternalRepositories(externalSources, { gh, sleep, carriedCommits });
|
|
170
210
|
const repositoryRows = canonicalRepositoryRows([...primary.rows, ...external.rows]);
|
|
171
211
|
const repositories = sourceSetObservation(repositoryRows, primary.expected + external.expected);
|
|
172
212
|
const repositoryKeys = new Set();
|
|
@@ -179,7 +219,7 @@ export function observeSourceUniverse({
|
|
|
179
219
|
repositoryKeys.add(repositoryKey);
|
|
180
220
|
storeKeys.add(storeKey);
|
|
181
221
|
}
|
|
182
|
-
const observedGists = observeGists(owner, { gh });
|
|
222
|
+
const observedGists = observeGists(owner, { gh, sleep });
|
|
183
223
|
const gists = sourceSetObservation(canonicalGistRows(observedGists.rows), observedGists.expected);
|
|
184
224
|
return canonicalSourceObservation({
|
|
185
225
|
schemaVersion: 1,
|
|
@@ -222,6 +262,8 @@ export function canonicalRepositoryRows(rows = []) {
|
|
|
222
262
|
isArchived: repo?.isArchived === true,
|
|
223
263
|
isDisabled: repo?.isDisabled === true,
|
|
224
264
|
...(repo?.upstreamUnavailable === true ? { upstreamUnavailable: true } : {}),
|
|
265
|
+
...(repo?.upstreamUnavailable === true && repo?.upstreamUnavailableReason ? { upstreamUnavailableReason: String(repo.upstreamUnavailableReason) } : {}),
|
|
266
|
+
...(repo?.upstreamUnavailable === true && repo?.upstreamUnavailableKind ? { upstreamUnavailableKind: String(repo.upstreamUnavailableKind) } : {}),
|
|
225
267
|
diskUsage: repo?.diskUsage,
|
|
226
268
|
updatedAt: repo?.updatedAt ?? null,
|
|
227
269
|
pushedAt: repo?.pushedAt ?? null,
|
|
@@ -325,7 +367,11 @@ function listGistsUnauthenticated(owner, curl) {
|
|
|
325
367
|
return pages;
|
|
326
368
|
}
|
|
327
369
|
|
|
328
|
-
export function observeGists(owner, { gh = runGh, curl = runCurl } = {}) {
|
|
370
|
+
export function observeGists(owner, { gh: rawGh = runGh, curl: rawCurl = runCurl, sleep = sleepSync } = {}) {
|
|
371
|
+
// A rate limit on either channel (measured 2026-09-26: the gist list) is retried, bounded, before the
|
|
372
|
+
// integration-token 403 is allowed to fall back to the anonymous list, which is itself rate-limited.
|
|
373
|
+
const gh = retryingRead(rawGh, sleep);
|
|
374
|
+
const curl = retryingRead(rawCurl, sleep);
|
|
329
375
|
let pages;
|
|
330
376
|
try {
|
|
331
377
|
const raw = gh(['api', `users/${owner}/gists?per_page=100`, '--paginate', '--slurp']);
|
|
@@ -414,8 +460,19 @@ export function classifyRepository(repo, evidence, exclusion = null, outcome = n
|
|
|
414
460
|
status = 'FAILED';
|
|
415
461
|
reasons.push(outcome.integrity);
|
|
416
462
|
}
|
|
417
|
-
if (repo.upstreamUnavailable === true)
|
|
463
|
+
if (repo.upstreamUnavailable === true) {
|
|
464
|
+
reasons.push(`upstream ${repo.fullName} ${repo.upstreamUnavailableReason || 'is no longer reachable (HTTP 404/410)'}; serving the content last ingested at ${upstreamSha}`);
|
|
465
|
+
}
|
|
418
466
|
const record = {};
|
|
467
|
+
// A source nobody could read tonight is never CURRENT: its bytes are pinned at the last commit the seed carried,
|
|
468
|
+
// and the row ships a carry record (missedUpstream null -- the upstream head is unknown, not merely different)
|
|
469
|
+
// that eligibleRepositoryStanding accepts only together with upstreamUnavailable. A `gone` source stays CURRENT:
|
|
470
|
+
// its upstream was READ and is final.
|
|
471
|
+
if (disposed && repo.upstreamUnavailableKind === 'transient' && status === 'CURRENT') {
|
|
472
|
+
status = 'STALE';
|
|
473
|
+
record.carry = { reason: TRANSIENT_CARRY_REASON, carriedSourceCommit: upstreamSha, missedUpstream: null,
|
|
474
|
+
attempts: 1, carriedCommittedAt: repo.defaultBranchRef?.target?.committedDate || null };
|
|
475
|
+
}
|
|
419
476
|
if (disposed && outcome?.carry && status === 'STALE') record.carry = { ...outcome.carry };
|
|
420
477
|
if (disposed && outcome?.failure && status === 'MISSING') record.failure = { ...outcome.failure };
|
|
421
478
|
return {
|
|
@@ -428,6 +485,7 @@ export function classifyRepository(repo, evidence, exclusion = null, outcome = n
|
|
|
428
485
|
disposition,
|
|
429
486
|
archived: repo.isArchived === true,
|
|
430
487
|
...(repo.upstreamUnavailable === true ? { upstreamUnavailable: true } : {}),
|
|
488
|
+
...(repo.upstreamUnavailable === true && repo.upstreamUnavailableKind ? { upstreamUnavailableKind: repo.upstreamUnavailableKind } : {}),
|
|
431
489
|
...(repo.forkDelta ? { forkDelta: { ...repo.forkDelta } } : {}),
|
|
432
490
|
upstream: { sha: upstreamSha, committedAt: repo.defaultBranchRef?.target?.committedDate || null,
|
|
433
491
|
pushedAt: repo.pushedAt, updatedAt: repo.updatedAt },
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// DISTINCT-FROM: kb/download-retry.mjs — that retries the customer updater's HTTPS stream by error code; this
|
|
2
|
+
// classifies and retries the pipeline's `gh`/`git` subprocess reads by what they printed.
|
|
3
|
+
//
|
|
4
|
+
// scripts/transient-gh.mjs — ONE rule for what a transient GitHub/API/git failure looks like, and ONE bounded
|
|
5
|
+
// retry for it, shared by every read in the corpus nightly (ADR-0091, fault classes in CONTRIBUTING.md).
|
|
6
|
+
//
|
|
7
|
+
// THE FAILURE THIS EXISTS TO END (measured on 38 corpus nights, 2026-09-21 .. 2026-10-18). Twenty-four nights
|
|
8
|
+
// failed; the causes were a gist API rate limit, a publisher that could not re-read the release it had just
|
|
9
|
+
// created, a canary refused one anonymous read, a git credential prompt on a clone -- each a single blip at a
|
|
10
|
+
// single `gh`/`git`/`fetch` call, each unretried, each the whole night for every customer. Before this module
|
|
11
|
+
// every call site either had no retry or its own private idea of "transient" (the canary's status set, the
|
|
12
|
+
// gist transport's regexes, the reconciler's exit-status classes). The rule is now written once:
|
|
13
|
+
//
|
|
14
|
+
// TRANSIENT a rate limit (primary or secondary), HTTP 429/408/5xx, a connection reset/refused/timed out, a
|
|
15
|
+
// DNS hiccup, a TLS handshake failure, a hung-up socket, git's credential prompt on an anonymous
|
|
16
|
+
// fetch ("could not read Username"), a partial transfer ("early EOF", "RPC failed").
|
|
17
|
+
// NOT HTTP 404/410 (gone: the ifGone carry in source-coverage.mjs owns it), a bare HTTP 403 from an
|
|
18
|
+
// authenticated gh (a permission answer, e.g. "Resource not accessible by integration"), "release
|
|
19
|
+
// not found" (the publisher's absence proof), a signature, digest or schema refusal, a missing
|
|
20
|
+
// binary (ENOENT). Those are answers, not weather; retrying them only delays the honest failure.
|
|
21
|
+
//
|
|
22
|
+
// Anonymous `fetch` reads from a shared runner address are the one place a bare 403 IS transient (GitHub's
|
|
23
|
+
// per-IP throttle answers 403): that status set, TRANSIENT_HTTP, lives here too so the canary and any other
|
|
24
|
+
// anonymous reader share it rather than restating it.
|
|
25
|
+
//
|
|
26
|
+
// WRITES ARE NEVER RETRIED BLINDLY. `gh release create` uploads assets; a retry after a half-finished create
|
|
27
|
+
// collides with the release it left behind. `retryingGh` retries reads and idempotent edits only; a create or
|
|
28
|
+
// any `gh api` POST/PUT/PATCH/DELETE runs exactly once and reports what happened.
|
|
29
|
+
|
|
30
|
+
/** Anonymous-fetch statuses a shared runner address sees for a moment (GitHub throttles anonymous reads with 403). */
|
|
31
|
+
export const TRANSIENT_HTTP = new Set([403, 408, 429, 500, 502, 503, 504]);
|
|
32
|
+
|
|
33
|
+
export const DEFAULT_ATTEMPTS = 3;
|
|
34
|
+
export const DEFAULT_BASE_DELAY_MS = 15_000;
|
|
35
|
+
/** The wait before retry `n` is `base * n`. RUVNET_TRANSIENT_RETRY_DELAY_MS is the test seam (the suite sets 0 so a
|
|
36
|
+
* fake gh that answers "gateway timeout" fails in milliseconds, not after the 45 s a real night would wait). */
|
|
37
|
+
export function defaultBaseDelayMs(fallback = DEFAULT_BASE_DELAY_MS) {
|
|
38
|
+
const seam = Number(process.env.RUVNET_TRANSIENT_RETRY_DELAY_MS);
|
|
39
|
+
return Number.isFinite(seam) && seam >= 0 && process.env.RUVNET_TRANSIENT_RETRY_DELAY_MS !== undefined ? seam : fallback;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
const TRANSIENT_TEXT = /\brate limit|secondary rate|abuse detection|HTTP (408|429|5\d\d)\b|\b(500|502|503|504) (Internal Server Error|Bad Gateway|Service Unavailable|Gateway Time-?out)|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|EPIPE|connection reset|connection refused|timed out|timeout|TLS handshake|socket hang up|could not resolve host|no such host|dial tcp|i\/o timeout|unexpected EOF|early EOF|RPC failed|remote end hung up|could not read Username|unable to access|temporarily unavailable|temporary failure/i;
|
|
43
|
+
|
|
44
|
+
/** What a failed subprocess said -- stderr, then stdout, then the spawn error -- as one string for classification. */
|
|
45
|
+
export function failureText(result) {
|
|
46
|
+
return [result?.stderr, result?.stdout, result?.error?.message].map((part) => String(part ?? '')).join('\n');
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** The one classifier: true when the text describes weather, not an answer. A missing binary is never weather. */
|
|
50
|
+
export function isTransientText(text) {
|
|
51
|
+
const message = String(text ?? '');
|
|
52
|
+
if (/\bENOENT\b/.test(message)) return false;
|
|
53
|
+
return TRANSIENT_TEXT.test(message);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/** A spawnSync-shaped result that failed transiently. */
|
|
57
|
+
export function isTransientResult(result) {
|
|
58
|
+
if (!result) return false;
|
|
59
|
+
if (!result.error && result.status === 0) return false;
|
|
60
|
+
if (result.error?.code === 'ENOENT') return false;
|
|
61
|
+
return isTransientText(failureText(result));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/** Blocks the thread for `ms`; the pipeline's gh/git callers are synchronous spawnSync loops. Injectable in tests. */
|
|
65
|
+
export function sleepSync(ms) {
|
|
66
|
+
if (ms > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
const delayFor = (attempt, baseDelayMs) => baseDelayMs * attempt;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Run `fn` (which THROWS on failure) up to `attempts` times while the failure is transient, waiting
|
|
73
|
+
* baseDelayMs * attempt between tries. The last error is rethrown unchanged, so callers keep their own messages.
|
|
74
|
+
*/
|
|
75
|
+
export function retryTransient(fn, { attempts = DEFAULT_ATTEMPTS, baseDelayMs = defaultBaseDelayMs(), sleep = sleepSync,
|
|
76
|
+
isTransient = (error) => isTransientText(error?.message), onRetry = () => {} } = {}) {
|
|
77
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
78
|
+
try {
|
|
79
|
+
return fn(attempt);
|
|
80
|
+
} catch (error) {
|
|
81
|
+
if (attempt >= attempts || !isTransient(error)) throw error;
|
|
82
|
+
onRetry({ attempt, error, delayMs: delayFor(attempt, baseDelayMs) });
|
|
83
|
+
sleep(delayFor(attempt, baseDelayMs));
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/** The async twin of retryTransient, for callers whose work (or sleep) is a promise. */
|
|
89
|
+
export async function retryTransientAsync(fn, { attempts = DEFAULT_ATTEMPTS, baseDelayMs = defaultBaseDelayMs(),
|
|
90
|
+
sleep = (ms) => new Promise((resolve) => { setTimeout(resolve, ms); }),
|
|
91
|
+
isTransient = (error) => isTransientText(error?.message), onRetry = () => {} } = {}) {
|
|
92
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
93
|
+
try {
|
|
94
|
+
return await fn(attempt);
|
|
95
|
+
} catch (error) {
|
|
96
|
+
if (attempt >= attempts || !isTransient(error)) throw error;
|
|
97
|
+
onRetry({ attempt, error, delayMs: delayFor(attempt, baseDelayMs) });
|
|
98
|
+
await sleep(delayFor(attempt, baseDelayMs));
|
|
99
|
+
}
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Run `fn` (which RETURNS a spawnSync-shaped result, never throws) up to `attempts` times while the result is a
|
|
105
|
+
* transient failure. Returns the last result, so callers that read `status`/`stderr` keep doing exactly that.
|
|
106
|
+
*/
|
|
107
|
+
export function retryTransientResult(fn, { attempts = DEFAULT_ATTEMPTS, baseDelayMs = defaultBaseDelayMs(),
|
|
108
|
+
sleep = sleepSync, onRetry = () => {} } = {}) {
|
|
109
|
+
let result;
|
|
110
|
+
for (let attempt = 1; ; attempt += 1) {
|
|
111
|
+
result = fn(attempt) || {};
|
|
112
|
+
if (attempt >= attempts || !isTransientResult(result)) return result;
|
|
113
|
+
onRetry({ attempt, result, delayMs: delayFor(attempt, baseDelayMs) });
|
|
114
|
+
sleep(delayFor(attempt, baseDelayMs));
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
const GH_WRITE_VERBS = new Set(['create', 'delete', 'upload', 'delete-asset']);
|
|
119
|
+
const REST_WRITE_FLAGS = new Set(['-f', '-F', '--input', '--field', '--raw-field']);
|
|
120
|
+
const METHOD_FLAGS = new Set(['-X', '--method']);
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Is this `gh` argument vector a write that must run exactly once? `release edit` is an idempotent PATCH and is
|
|
124
|
+
* NOT a write here: re-sending the same flags after a timeout converges on the same state. `gh api` is a write
|
|
125
|
+
* when it names a non-GET method or carries a field flag (gh then sends POST); `gh api graphql` always POSTs.
|
|
126
|
+
*/
|
|
127
|
+
export function isGhWrite(args) {
|
|
128
|
+
const argv = (args || []).map(String);
|
|
129
|
+
if (argv[0] === 'api') {
|
|
130
|
+
const methodAt = argv.findIndex((token) => METHOD_FLAGS.has(token));
|
|
131
|
+
if (methodAt >= 0 && String(argv[methodAt + 1] || '').toUpperCase() !== 'GET') return true;
|
|
132
|
+
if (argv[1] === 'graphql') return true;
|
|
133
|
+
return argv.some((token) => REST_WRITE_FLAGS.has(token));
|
|
134
|
+
}
|
|
135
|
+
return argv[0] === 'release' && GH_WRITE_VERBS.has(argv[1]);
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* A `gh` runner over a spawnSync-shaped `run` that retries transient failures of READS and idempotent edits,
|
|
140
|
+
* bounded, and runs a write exactly once. `prefix` is the RUVNET_GH_SCRIPT test seam (scripts/release.mjs).
|
|
141
|
+
*/
|
|
142
|
+
export function retryingGh({ run, command = 'gh', prefix = [], options = {}, sleep = sleepSync, attempts = DEFAULT_ATTEMPTS,
|
|
143
|
+
baseDelayMs = defaultBaseDelayMs(), onRetry = () => {} } = {}) {
|
|
144
|
+
if (typeof run !== 'function') throw new Error('retryingGh needs a spawnSync-shaped run function');
|
|
145
|
+
return (args) => {
|
|
146
|
+
const once = () => run(command, [...prefix, ...args], options) || {};
|
|
147
|
+
if (isGhWrite(args)) return once();
|
|
148
|
+
return retryTransientResult(once, { attempts, baseDelayMs, sleep, onRetry: (event) => onRetry({ ...event, args }) });
|
|
149
|
+
};
|
|
150
|
+
}
|