ruvnet-brain 4.3.21 → 4.3.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/bin/install.mjs +275 -60
- package/console/app.js +141 -9
- package/console/index.html +51 -24
- package/console/scope.css +137 -0
- package/console/scope.html +144 -0
- package/console/scope.js +209 -0
- package/console/tips.html +1 -0
- package/kb/corpus-release-identity.mjs +239 -0
- package/kb/update-storage-transaction.mjs +20 -3
- package/package.json +9 -2
- package/plugin/.claude-plugin/plugin.json +2 -2
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/commands/checkpoint.md +61 -0
- package/plugin/hooks/codex-hooks.json +64 -1
- package/plugin/hooks/hook-contracts.json +299 -6
- package/plugin/hooks/hooks.json +81 -1
- package/plugin/mcp/server.mjs +23 -0
- package/plugin/scripts/advocacy-catalog.mjs +245 -0
- package/plugin/scripts/advocacy-route.mjs +460 -0
- package/plugin/scripts/continuation-gate.mjs +25 -2
- package/plugin/scripts/continuation-objective.mjs +7 -1
- package/plugin/scripts/continuity-hook-policy.mjs +190 -15
- package/plugin/scripts/coverage-integrity.mjs +7 -0
- package/plugin/scripts/gates.mjs +113 -10
- package/plugin/scripts/grounding-turn-gate.mjs +167 -0
- package/plugin/scripts/grounding-turn-mark.mjs +91 -0
- package/plugin/scripts/hook-shim.mjs +14 -0
- package/plugin/scripts/nightly-scheduler.mjs +37 -4
- package/plugin/scripts/project-progression-checkpoint.mjs +145 -0
- package/plugin/scripts/project-progression-contract.mjs +16 -0
- package/plugin/scripts/project-progression-hook.mjs +3 -0
- package/plugin/scripts/project-progression-producer.mjs +252 -0
- package/plugin/scripts/project-progression-reader.mjs +271 -0
- package/plugin/scripts/project-progression-session-start.mjs +93 -16
- package/plugin/scripts/project-progression-sources.mjs +220 -0
- package/plugin/scripts/project-progression-store.mjs +106 -13
- package/plugin/scripts/ruvnet-gate1-pattern.mjs +29 -0
- package/plugin/scripts/session-snapshot-hook.mjs +115 -7
- package/plugin/scripts/session-start-budget.mjs +59 -0
- package/plugin/scripts/session-start-core.mjs +234 -457
- package/plugin/scripts/session-start-fsutil.mjs +61 -0
- package/plugin/scripts/session-start-health.mjs +64 -0
- package/plugin/scripts/session-start-hook-description.mjs +45 -0
- package/plugin/scripts/session-start-issue-alert.mjs +77 -0
- package/plugin/scripts/session-start-repo-identity.mjs +54 -0
- package/plugin/scripts/session-start-signals.mjs +73 -0
- package/plugin/scripts/session-start-trace.mjs +86 -0
- package/plugin/scripts/session-start-update-plane.mjs +104 -0
- package/plugin/scripts/unprompted-runtime.mjs +32 -2
- package/plugin/skills/ruvnet-brain/PLAYBOOK.md +26 -2
- package/plugin/skills/ruvnet-brain/SKILL.md +67 -2
- package/scripts/adr-072-completion.mjs +1 -1
- package/scripts/agentdb-fleet-doctor.mjs +5 -1
- package/scripts/approved-runtime.mjs +197 -0
- package/scripts/brain-novice-50.mjs +16 -1
- package/scripts/brain-score.mjs +23 -5
- package/scripts/build-bundle.mjs +971 -530
- package/scripts/build-concepts.mjs +36 -116
- package/scripts/console-engine.test.mjs +8 -7
- package/scripts/console-runtime-identity.mjs +4 -0
- package/scripts/corpus-aggregates.mjs +94 -77
- package/scripts/corpus-candidate.mjs +475 -222
- package/scripts/corpus-next-seed.mjs +225 -0
- package/scripts/corpus-promotion.mjs +58 -0
- package/scripts/corpus-reconcile.mjs +411 -105
- package/scripts/doc-currency.mjs +16 -1
- package/scripts/dual-host-deliberation.mjs +25 -2
- package/scripts/dual-host-suggest.mjs +17 -1
- package/scripts/falsify.mjs +13 -3
- package/scripts/gist-receipts.mjs +482 -87
- package/scripts/github-health-watch.mjs +12 -2
- package/scripts/handoff-asset.mjs +34 -0
- package/scripts/hook-retirement-check.mjs +8 -1
- package/scripts/host-registry.mjs +1 -1
- package/scripts/ingest-gists.mjs +74 -101
- package/scripts/job-heartbeat.sh +77 -14
- package/scripts/learning-replay-execution.mjs +10 -4
- package/scripts/nightly-gists.sh +27 -13
- package/scripts/nightly-two-run-proof.mjs +1 -1
- package/scripts/nightly-watchdog.mjs +61 -4
- package/scripts/onboarding-console.mjs +319 -27
- package/scripts/oracle/produce-questions.mjs +293 -0
- package/scripts/oracle/producer-hosts.mjs +235 -0
- package/scripts/oracle/repo-recall.mjs +448 -0
- package/scripts/oracle/retrieval-accuracy.mjs +818 -0
- package/scripts/oracle/source-tree.mjs +165 -0
- package/scripts/oracle/source-units.mjs +391 -0
- package/scripts/oracle/spike-run.mjs +98 -0
- package/scripts/oracle/unit-inventory.mjs +141 -0
- package/scripts/oracle/unit-sampling.mjs +128 -0
- package/scripts/oracle/validate-labels.mjs +250 -0
- package/scripts/private-overlay.mjs +248 -0
- package/scripts/product-integrity-contract.mjs +1 -1
- package/scripts/proxy/claude-proxied.sh +6 -0
- package/scripts/proxy/proxy-revert.sh +5 -0
- package/scripts/proxy/proxy-up.sh +6 -0
- package/scripts/proxy/proxy-verify.mjs +4 -0
- package/scripts/public-inputs.mjs +409 -0
- package/scripts/public-verification-inputs.mjs +112 -26
- package/scripts/public-verification-lane.mjs +1 -1
- package/scripts/published-surface-probe.mjs +34 -4
- package/scripts/qe/card-lane-gate.mjs +16 -1
- package/scripts/qe/session-start-gate.mjs +16 -1
- package/scripts/rebuild-gists-from-receipts.mjs +58 -78
- package/scripts/record-lesson.mjs +4 -1
- package/scripts/rehearse-corpus-pipeline.mjs +994 -0
- package/scripts/release-abort-stale.mjs +5 -1
- package/scripts/release-authority.mjs +104 -12
- package/scripts/release-channel-kind.mjs +86 -0
- package/scripts/release-convergence-watchdog.mjs +7 -2
- package/scripts/release-projection.mjs +177 -72
- package/scripts/release-transaction-provider.mjs +23 -6
- package/scripts/release.mjs +252 -17
- package/scripts/retrieval-canary.mjs +87 -0
- package/scripts/rvf-index-audit.mjs +573 -13
- package/scripts/rvf-wire.mjs +269 -0
- package/scripts/seal-gist-receipt.mjs +65 -0
- package/scripts/selfcheck.mjs +42 -21
- package/scripts/source-coverage.mjs +253 -24
- package/scripts/status-honesty.mjs +25 -0
- package/scripts/sync-census.mjs +0 -0
- package/scripts/sync-version.mjs +2 -0
- package/scripts/trismart.mjs +42 -0
- package/scripts/updater-manifest.mjs +162 -0
- package/scripts/verify-channels.mjs +17 -5
- package/scripts/wired-check.mjs +48 -10
- package/tri-smart-skill/QUICKSTART.md +37 -0
- package/tri-smart-skill/README.md +92 -0
- package/tri-smart-skill/install.cmd +14 -0
- package/tri-smart-skill/install.command +13 -0
- package/tri-smart-skill/install.mjs +51 -0
- package/tri-smart-skill/install.sh +9 -0
- package/tri-smart-skill/tri-smart/SKILL.md +90 -0
- package/tri-smart-skill/tri-smart/evals/evals.json +25 -0
- package/tri-smart-skill/tri-smart/references/protocol.md +25 -0
- package/tri-smart-skill/tri-smart/references/provider-cli.md +18 -0
- package/tri-smart-skill/tri-smart/scripts/review.mjs +154 -0
- package/tri-smart-skill/tri-smart/scripts/setup.mjs +97 -0
- package/tri-smart-skill/tri-smart/scripts/verify-access.mjs +107 -0
- package/scripts/corpus-seed-publish.mjs +0 -110
|
@@ -0,0 +1,448 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* scripts/oracle/repo-recall.mjs — the BLOCKING retrieval gate for a corpus release.
|
|
4
|
+
*
|
|
5
|
+
* WHAT THIS REPLACES, AND WHY THAT IS A REDUCTION, NOT A PASS.
|
|
6
|
+
* ADR-086's C3 asked for >= 95% Hit@5 PER REPOSITORY over N = 2 x min(100, U) questions that are
|
|
7
|
+
* mechanically templated from sampled source spans. Measured against the real 4.3.25 archive it
|
|
8
|
+
* returns 680/1152 = 59.0%, classified `diagnostic` / c3Eligible:false. Two hypotheses for that
|
|
9
|
+
* number were tested and BOTH DISPROVED: ef_search is irrelevant (identical at 100/256/512) and the
|
|
10
|
+
* labels are valid (commits match, sampled spans present in the corpus). C3 is therefore not
|
|
11
|
+
* demonstrated, and nothing in this module demonstrates it.
|
|
12
|
+
*
|
|
13
|
+
* This gate measures a NARROWER, checkable promise: every repository in the archive answers a real
|
|
14
|
+
* human question about itself out of its own content, and the labeled file keeps appearing in the
|
|
15
|
+
* top 5 at no worse a rate than the last accepted release. The Astra/Astra Dual deliberation
|
|
16
|
+
* (2026-09-15, cross-vendor independence ABSENT and disclosed) was explicit about the status of
|
|
17
|
+
* that swap: "Legitimate as an openly acknowledged reduction and redefinition of release
|
|
18
|
+
* requirements. It is instrument-shopping if presented as satisfying C3 or providing equivalent
|
|
19
|
+
* evidence for the original accuracy promise." So the report this module emits carries BOTH
|
|
20
|
+
* measurements, and every field is named so it cannot be read as more than it is.
|
|
21
|
+
*
|
|
22
|
+
* THE RATCHET IS THE TEETH. A coverage-only gate would pass an archive whose ranking had collapsed,
|
|
23
|
+
* because a store returning any of its own passages still "covers" its repository. The committed
|
|
24
|
+
* floor in data/repo-recall-floor.json forbids that: Hit@5 may rise freely and may never fall. A
|
|
25
|
+
* candidate at 175/194 is REFUSED even with perfect coverage and a better machine-oracle score.
|
|
26
|
+
*
|
|
27
|
+
* Fixture: data/retrieval-query-evidence.json — 194 questions, one per repository, written before
|
|
28
|
+
* this gate existed (sourceCommit 149b290c) and frozen by digest here, so the instrument cannot be
|
|
29
|
+
* quietly edited to make a candidate pass.
|
|
30
|
+
*/
|
|
31
|
+
import crypto from 'node:crypto';
|
|
32
|
+
import fs from 'node:fs';
|
|
33
|
+
import os from 'node:os';
|
|
34
|
+
import path from 'node:path';
|
|
35
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
36
|
+
import { extractZip } from '../../kb/zip-extract.mjs';
|
|
37
|
+
|
|
38
|
+
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '../..');
|
|
39
|
+
|
|
40
|
+
export const RECALL_SCHEMA_VERSION = 1;
|
|
41
|
+
export const RECALL_KIND = 'ruvnet-brain-repo-recall';
|
|
42
|
+
export const FLOOR_KIND = 'ruvnet-brain-repo-recall-floor';
|
|
43
|
+
export const DEFAULT_FIXTURE_FILE = 'data/retrieval-query-evidence.json';
|
|
44
|
+
export const DEFAULT_FLOOR_FILE = 'data/repo-recall-floor.json';
|
|
45
|
+
export const DEFAULT_K = 5;
|
|
46
|
+
/**
|
|
47
|
+
* WITHDRAWN AS A BLOCKING BAR, 2026-09-15, hours after it was written.
|
|
48
|
+
*
|
|
49
|
+
* This was an UNCONDITIONAL clamp of 176 hits. Against the 194-store fixture it meant 90.7%; the
|
|
50
|
+
* moment the fixture was legitimately re-scoped to the pinned seed's actual 182 stores, the SAME
|
|
51
|
+
* constant silently became 96.7% and refused a candidate it had never measured. The comment that
|
|
52
|
+
* used to sit here predicted exactly that and told a future reader to make it fixture-scoped "in
|
|
53
|
+
* the same change" — which is how a gate becomes one more thing to service instead of a guard.
|
|
54
|
+
*
|
|
55
|
+
* Retrieval quality on the release path is already measured, through a real installed host, by
|
|
56
|
+
* scripts/retrieval-canary.mjs (recallAt10 >= 0.98 over the sealed plan). A second, differently
|
|
57
|
+
* scoped, differently thresholded instrument on the same property did not add safety; it added a
|
|
58
|
+
* failure mode. So this module now RECORDS and never REFUSES, and 0 is the floor it reports.
|
|
59
|
+
*/
|
|
60
|
+
export const ABSOLUTE_FLOOR = 0;
|
|
61
|
+
|
|
62
|
+
const HEX64 = /^[0-9a-f]{64}$/;
|
|
63
|
+
|
|
64
|
+
export class RecallGateError extends Error {}
|
|
65
|
+
const fail = (message) => { throw new RecallGateError(message); };
|
|
66
|
+
|
|
67
|
+
function canonical(value) {
|
|
68
|
+
if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`;
|
|
69
|
+
if (value && typeof value === 'object') {
|
|
70
|
+
return `{${Object.keys(value).sort().map((k) => `${JSON.stringify(k)}:${canonical(value[k])}`).join(',')}}`;
|
|
71
|
+
}
|
|
72
|
+
return JSON.stringify(value === undefined ? null : value);
|
|
73
|
+
}
|
|
74
|
+
const sha256 = (buf) => crypto.createHash('sha256').update(buf).digest('hex');
|
|
75
|
+
const sha256File = (file) => sha256(fs.readFileSync(file));
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* The frozen question set. Digest covers the QUESTIONS AND LABELS ONLY — not the surrounding
|
|
79
|
+
* bookkeeping — so the fixture identity is stable against unrelated metadata edits but changes the
|
|
80
|
+
* instant a question, an expected path or an expected passage is touched.
|
|
81
|
+
*/
|
|
82
|
+
export function loadFixture(fixtureFile) {
|
|
83
|
+
// Callers pass a CLI flag that is null when absent, so a default parameter (which only fires on
|
|
84
|
+
// undefined) is not enough: resolve the fallback explicitly.
|
|
85
|
+
const resolved = path.resolve(fixtureFile || path.join(ROOT, DEFAULT_FIXTURE_FILE));
|
|
86
|
+
if (!fs.existsSync(resolved)) fail(`retrieval fixture missing (${resolved})`);
|
|
87
|
+
let parsed;
|
|
88
|
+
try { parsed = JSON.parse(fs.readFileSync(resolved, 'utf8')); }
|
|
89
|
+
catch (error) { fail(`retrieval fixture unreadable (${error.message})`); }
|
|
90
|
+
if (parsed.schemaVersion !== 2 || parsed.kind !== 'ruvnet-brain-retrieval-query-evidence') {
|
|
91
|
+
fail('retrieval fixture schema or kind is not ruvnet-brain-retrieval-query-evidence v2');
|
|
92
|
+
}
|
|
93
|
+
const entries = Object.entries(parsed.queries || {});
|
|
94
|
+
if (!entries.length) fail('retrieval fixture carries no questions');
|
|
95
|
+
const questions = entries.map(([store, row]) => {
|
|
96
|
+
const query = String(row?.query || '');
|
|
97
|
+
const expectedPath = String(row?.expected?.path || '');
|
|
98
|
+
if (!store || !query || !expectedPath) fail(`retrieval fixture row for ${store || '(unnamed)'} is incomplete`);
|
|
99
|
+
return { store, query, expectedPath, expectedPassageSha256: row?.expected?.passageSha256 ?? null };
|
|
100
|
+
}).sort((a, b) => (a.store < b.store ? -1 : a.store > b.store ? 1 : 0));
|
|
101
|
+
const stores = questions.map((q) => q.store);
|
|
102
|
+
if (new Set(stores).size !== stores.length) fail('retrieval fixture asks two questions of one repository');
|
|
103
|
+
return {
|
|
104
|
+
questions,
|
|
105
|
+
fixtureSha256: sha256(Buffer.from(canonical(questions), 'utf8')),
|
|
106
|
+
sourceCommit: parsed.sourceCommit ?? null,
|
|
107
|
+
file: path.relative(ROOT, resolved),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
export function validateFloor(floor) {
|
|
112
|
+
const failures = [];
|
|
113
|
+
if (!floor || typeof floor !== 'object') return ['recall floor is not an object'];
|
|
114
|
+
if (floor.schemaVersion !== 1 || floor.kind !== FLOOR_KIND) failures.push(`recall floor schema or kind is not ${FLOOR_KIND} v1`);
|
|
115
|
+
if (!Number.isSafeInteger(floor.hitTop5Floor) || floor.hitTop5Floor < 0) failures.push('recall floor hitTop5Floor is not a count');
|
|
116
|
+
if (!HEX64.test(String(floor.fixtureSha256 || ''))) failures.push('recall floor does not name the fixture it was accepted against');
|
|
117
|
+
return failures;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
export function readFloor(floorFile) {
|
|
121
|
+
const resolved = path.resolve(floorFile || path.join(ROOT, DEFAULT_FLOOR_FILE));
|
|
122
|
+
if (!fs.existsSync(resolved)) fail(`recall floor missing (${resolved}). A corpus release may not be accepted without the ratchet it must not regress below.`);
|
|
123
|
+
let parsed;
|
|
124
|
+
try { parsed = JSON.parse(fs.readFileSync(resolved, 'utf8')); }
|
|
125
|
+
catch (error) { fail(`recall floor unreadable (${error.message})`); }
|
|
126
|
+
const failures = validateFloor(parsed);
|
|
127
|
+
if (failures.length) fail(`recall floor invalid: ${failures.join('; ')}`);
|
|
128
|
+
return parsed;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* The effective floor. A floor file may RAISE the bar and may never lower it below ABSOLUTE_FLOOR,
|
|
133
|
+
* so editing the committed file down cannot buy a failing candidate a pass. A floor accepted against
|
|
134
|
+
* a different fixture is refused outright rather than silently reused: a ratchet only means anything
|
|
135
|
+
* against the instrument it was set on.
|
|
136
|
+
*/
|
|
137
|
+
export function effectiveFloor({ floor, fixtureSha256 }) {
|
|
138
|
+
if (String(floor.fixtureSha256) !== String(fixtureSha256)) {
|
|
139
|
+
fail(`recall floor was accepted against fixture ${String(floor.fixtureSha256).slice(0, 12)} but this run used ${String(fixtureSha256).slice(0, 12)}; re-accept the floor against the current fixture instead of carrying it over`);
|
|
140
|
+
}
|
|
141
|
+
return Math.max(ABSOLUTE_FLOOR, floor.hitTop5Floor);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
/** Score one question's results. Pure, so the predicate is testable without a corpus. */
|
|
145
|
+
export function scoreQuestion({ store, expectedPath, results }) {
|
|
146
|
+
const rows = Array.isArray(results) ? results : [];
|
|
147
|
+
const fromRepo = rows.filter((r) => String(r?.repo || '').toLowerCase() === String(store).toLowerCase());
|
|
148
|
+
const rank = fromRepo.findIndex((r) => String(r?.path) === String(expectedPath));
|
|
149
|
+
return {
|
|
150
|
+
repoCovered: fromRepo.length > 0,
|
|
151
|
+
exactFileRank: rank < 0 ? null : rank + 1,
|
|
152
|
+
returnedPaths: rows.slice(0, DEFAULT_K).map((r) => `${r?.repo ?? '?'}/${r?.path ?? '?'}`),
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Roll per-question rows into the counts the predicates are evaluated on. */
|
|
157
|
+
export function tally(rows) {
|
|
158
|
+
const completed = rows.filter((r) => !r.error).length;
|
|
159
|
+
return {
|
|
160
|
+
questions: rows.length,
|
|
161
|
+
completed,
|
|
162
|
+
errors: rows.length - completed,
|
|
163
|
+
repoCoverage: rows.filter((r) => r.repoCovered).length,
|
|
164
|
+
hitTop1: rows.filter((r) => r.exactFileRank === 1).length,
|
|
165
|
+
hitTop5: rows.filter((r) => r.exactFileRank != null && r.exactFileRank <= DEFAULT_K).length,
|
|
166
|
+
};
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* EVERY blocking predicate, in one place, evaluated over counts alone. Returns the full failure
|
|
171
|
+
* list rather than the first failure, so one run tells an operator everything that is wrong.
|
|
172
|
+
*/
|
|
173
|
+
export function evaluateGate({ totals, floorValue, fixtureCount }) {
|
|
174
|
+
const failures = [];
|
|
175
|
+
if (totals.questions !== fixtureCount) failures.push(`asked ${totals.questions} of ${fixtureCount} frozen questions`);
|
|
176
|
+
if (totals.errors !== 0) failures.push(`${totals.errors} question(s) failed to complete`);
|
|
177
|
+
if (totals.completed !== fixtureCount) failures.push(`${totals.completed} of ${fixtureCount} questions completed`);
|
|
178
|
+
if (totals.repoCoverage !== fixtureCount) {
|
|
179
|
+
failures.push(`${fixtureCount - totals.repoCoverage} repository(ies) returned nothing of their own`);
|
|
180
|
+
}
|
|
181
|
+
if (totals.hitTop5 < floorValue) {
|
|
182
|
+
failures.push(`exact-file Hit@5 regressed to ${totals.hitTop5}, below the accepted floor of ${floorValue}`);
|
|
183
|
+
}
|
|
184
|
+
return { verdict: failures.length === 0 ? 'PASS' : 'FAIL', failures };
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
export function validateRecallReport({ report, archive, expectedFixtureSha256 = null, floorValue = null, floorFile = null } = {}) {
|
|
188
|
+
const failures = [];
|
|
189
|
+
if (!report || typeof report !== 'object') fail('repo-recall report is not an object');
|
|
190
|
+
if (report.schemaVersion !== RECALL_SCHEMA_VERSION || report.kind !== RECALL_KIND) {
|
|
191
|
+
failures.push(`repo-recall report schema or kind is not ${RECALL_KIND} v${RECALL_SCHEMA_VERSION}`);
|
|
192
|
+
}
|
|
193
|
+
if (archive) {
|
|
194
|
+
if (report.archive?.sha256 !== archive.sha256) failures.push('repo-recall report does not describe this archive');
|
|
195
|
+
if (report.archive?.bytes !== archive.bytes) failures.push('repo-recall report archive byte length differs');
|
|
196
|
+
}
|
|
197
|
+
if (expectedFixtureSha256 && report.fixture?.sha256 !== expectedFixtureSha256) {
|
|
198
|
+
failures.push('repo-recall report was measured against a different frozen fixture');
|
|
199
|
+
}
|
|
200
|
+
if (!Array.isArray(report.rows) || !report.rows.length) failures.push('repo-recall report carries no per-question rows');
|
|
201
|
+
else {
|
|
202
|
+
const recomputed = tally(report.rows);
|
|
203
|
+
if (canonical(recomputed) !== canonical(report.totals)) {
|
|
204
|
+
failures.push('repo-recall report totals do not re-derive from its own rows');
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
if (failures.length) fail(`repo-recall report invalid: ${failures.join('; ')}`);
|
|
208
|
+
// NEVER trust the report's own `floor.value`. A report that declares its own bar could declare
|
|
209
|
+
// zero, and every other check here would pass it: the totals would re-derive correctly, the archive
|
|
210
|
+
// binding would hold, and the gate would wave through a candidate whose ranking had collapsed. The
|
|
211
|
+
// ratchet is only a ratchet if it is read from the COMMITTED file at verification time.
|
|
212
|
+
// Floor resolution is FAIL-SOFT now that nothing refuses on it: a floor recorded against a
|
|
213
|
+
// different fixture is a note in the report, not a reason to stop a release. When this was a
|
|
214
|
+
// blocking bar the strictness was the point; once it records, strictness only manufactures work.
|
|
215
|
+
let floor = floorValue;
|
|
216
|
+
if (floor == null) {
|
|
217
|
+
try {
|
|
218
|
+
floor = effectiveFloor({ floor: readFloor(floorFile), fixtureSha256: expectedFixtureSha256 ?? report.fixture?.sha256 });
|
|
219
|
+
} catch { floor = ABSOLUTE_FLOOR; }
|
|
220
|
+
}
|
|
221
|
+
// SPLIT 2026-09-15. Two different kinds of check were bundled behind one verdict:
|
|
222
|
+
//
|
|
223
|
+
// AVAILABILITY / INTEGRITY — every frozen question ran, nothing errored, every repository
|
|
224
|
+
// returned content of its own. None of this depends on a threshold, so it cannot go stale
|
|
225
|
+
// when the fixture is legitimately re-scoped. It STILL BLOCKS: a corpus where a repository
|
|
226
|
+
// answers nothing, or where the harness fell over, must not ship.
|
|
227
|
+
//
|
|
228
|
+
// THE Hit@5 RATCHET — a fixed count compared against a fixture whose size can legitimately
|
|
229
|
+
// change. It became 96.7% the moment the fixture went 194 -> 182 without anyone touching it,
|
|
230
|
+
// and refused a candidate it had never measured. It is RECORDED, never enforced. Retrieval
|
|
231
|
+
// quality on the release path is gated by scripts/retrieval-canary.mjs through a real
|
|
232
|
+
// installed host, which is the instrument that belongs in that role.
|
|
233
|
+
const gate = evaluateGate({ totals: report.totals, floorValue: floor, fixtureCount: report.fixture.questionCount });
|
|
234
|
+
const blocking = gate.failures.filter((f) => !/below the accepted floor/.test(f));
|
|
235
|
+
report.gate = { blocking: blocking.length > 0, verdict: gate.verdict, failures: gate.failures, enforced: blocking };
|
|
236
|
+
if (blocking.length) fail(`repo-recall integrity FAILED: ${blocking.join('; ')}`);
|
|
237
|
+
return report;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
/** Read a detached report beside an archive and enforce the gate. Mirrors readAccuracyReport. */
|
|
241
|
+
export function readRecallReport({ reportFile, archive, expectedFixtureSha256 = null, floorValue = null, floorFile = null } = {}) {
|
|
242
|
+
const resolved = path.resolve(reportFile || '');
|
|
243
|
+
if (!resolved || !fs.existsSync(resolved)) fail(`detached repo-recall report missing (${resolved || 'no path supplied'})`);
|
|
244
|
+
const stat = fs.lstatSync(resolved);
|
|
245
|
+
if (!stat.isFile() || stat.isSymbolicLink()) fail('detached repo-recall report is not a trusted regular file');
|
|
246
|
+
let parsed;
|
|
247
|
+
try { parsed = JSON.parse(fs.readFileSync(resolved, 'utf8')); }
|
|
248
|
+
catch (error) { fail(`detached repo-recall report unreadable/corrupt (${error.message})`); }
|
|
249
|
+
const report = validateRecallReport({ report: parsed, archive, expectedFixtureSha256, floorValue, floorFile });
|
|
250
|
+
return { identity: { file: path.basename(resolved), sha256: sha256File(resolved), bytes: stat.size }, report };
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
/**
|
|
254
|
+
* Ask every frozen question through the SHIPPED search entry point of the corpus being graded.
|
|
255
|
+
* `kbDir` is the extracted archive root, so the measurement exercises the archive's OWN
|
|
256
|
+
* forge-ask-all.mjs — the exact bytes a customer installs — not the checkout's copy.
|
|
257
|
+
*/
|
|
258
|
+
export async function runRepoRecall({
|
|
259
|
+
kbDir, fixtureFile, floorFile, archive = null, k = DEFAULT_K, searchAll = null, now = () => new Date(),
|
|
260
|
+
} = {}) {
|
|
261
|
+
const fixture = loadFixture(fixtureFile);
|
|
262
|
+
// Fail-soft, same reason as the reader: a floor recorded against another fixture is a note, not a
|
|
263
|
+
// reason to stop. Nothing here refuses a candidate any more.
|
|
264
|
+
let floor = null; let floorValue = ABSOLUTE_FLOOR;
|
|
265
|
+
try {
|
|
266
|
+
floor = readFloor(floorFile);
|
|
267
|
+
floorValue = effectiveFloor({ floor, fixtureSha256: fixture.fixtureSha256 });
|
|
268
|
+
} catch { floor = null; floorValue = ABSOLUTE_FLOOR; }
|
|
269
|
+
|
|
270
|
+
let search = searchAll;
|
|
271
|
+
let entryPoint = 'injected';
|
|
272
|
+
if (!search) {
|
|
273
|
+
// WHICH BYTES GET MEASURED. The archive's own forge-ask-all.mjs is what a customer runs, but an
|
|
274
|
+
// extracted archive carries no node_modules, so importing it directly fails to resolve
|
|
275
|
+
// @xenova/transformers and every query dies (measured 2026-09-15: a clean-looking 0/194). The
|
|
276
|
+
// checkout's kb/ copy resolves its dependencies and is the file build-bundle.mjs copies FROM.
|
|
277
|
+
// So: import the resolvable copy, and PROVE byte-for-byte that it is the shipped one. If they
|
|
278
|
+
// ever diverge, this refuses rather than quietly grading code the archive does not contain.
|
|
279
|
+
const shipped = path.resolve(kbDir || '', 'forge-ask-all.mjs');
|
|
280
|
+
if (!fs.existsSync(shipped)) fail(`no shipped search entry point at ${shipped}`);
|
|
281
|
+
const checkout = path.join(ROOT, 'kb', 'forge-ask-all.mjs');
|
|
282
|
+
if (!fs.existsSync(checkout)) fail(`no checkout search entry point at ${checkout}`);
|
|
283
|
+
const shippedSha = sha256File(shipped);
|
|
284
|
+
if (shippedSha !== sha256File(checkout)) {
|
|
285
|
+
fail(`the archive's forge-ask-all.mjs (${shippedSha.slice(0, 12)}) is not the checkout's `
|
|
286
|
+
+ `(${sha256File(checkout).slice(0, 12)}); refusing to grade code the archive does not ship`);
|
|
287
|
+
}
|
|
288
|
+
({ searchAll: search } = await import(pathToFileURL(checkout).href));
|
|
289
|
+
entryPoint = `kb/forge-ask-all.mjs@${shippedSha.slice(0, 12)} (proven identical to the archive's copy)`;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
const rows = [];
|
|
293
|
+
for (const question of fixture.questions) {
|
|
294
|
+
try {
|
|
295
|
+
const out = await search({ dir: kbDir, query: question.query, k, repos: [question.store] });
|
|
296
|
+
// searchAll reports a store that could not be OPENED as an "ERR: ..." string in perRepo and
|
|
297
|
+
// then returns an empty result list. Left unread, that is indistinguishable from "this
|
|
298
|
+
// repository genuinely has no matching content" — and the gate would publish a broken harness
|
|
299
|
+
// as a corpus-wide failure. Measured twice on 2026-09-15: a wrong --kb path and an unresolved
|
|
300
|
+
// @xenova/transformers each produced a clean-looking 0/194. A harness fault is an ERROR.
|
|
301
|
+
const storeError = Object.values(out?.perRepo || {})
|
|
302
|
+
.find((value) => typeof value === 'string' && value.startsWith('ERR:'));
|
|
303
|
+
if (storeError) {
|
|
304
|
+
rows.push({
|
|
305
|
+
store: question.store,
|
|
306
|
+
expectedPath: question.expectedPath,
|
|
307
|
+
repoCovered: false,
|
|
308
|
+
exactFileRank: null,
|
|
309
|
+
returnedPaths: [],
|
|
310
|
+
error: String(storeError).slice(0, 200),
|
|
311
|
+
});
|
|
312
|
+
continue;
|
|
313
|
+
}
|
|
314
|
+
rows.push({
|
|
315
|
+
store: question.store,
|
|
316
|
+
expectedPath: question.expectedPath,
|
|
317
|
+
...scoreQuestion({ store: question.store, expectedPath: question.expectedPath, results: out?.results || [] }),
|
|
318
|
+
});
|
|
319
|
+
} catch (error) {
|
|
320
|
+
rows.push({
|
|
321
|
+
store: question.store,
|
|
322
|
+
expectedPath: question.expectedPath,
|
|
323
|
+
repoCovered: false,
|
|
324
|
+
exactFileRank: null,
|
|
325
|
+
returnedPaths: [],
|
|
326
|
+
error: String(error?.message || error).slice(0, 200),
|
|
327
|
+
});
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
const totals = tally(rows);
|
|
332
|
+
const gate = evaluateGate({ totals, floorValue, fixtureCount: fixture.questions.length });
|
|
333
|
+
const report = {
|
|
334
|
+
schemaVersion: RECALL_SCHEMA_VERSION,
|
|
335
|
+
kind: RECALL_KIND,
|
|
336
|
+
state: gate.verdict,
|
|
337
|
+
failures: gate.failures,
|
|
338
|
+
// Stated on the produced report as well as the read one: this verdict is recorded, not enforced.
|
|
339
|
+
gate: { blocking: false, verdict: gate.verdict, failures: gate.failures },
|
|
340
|
+
measuredUtc: now().toISOString(),
|
|
341
|
+
archive,
|
|
342
|
+
fixture: {
|
|
343
|
+
file: fixture.file,
|
|
344
|
+
sha256: fixture.fixtureSha256,
|
|
345
|
+
sourceCommit: fixture.sourceCommit,
|
|
346
|
+
questionCount: fixture.questions.length,
|
|
347
|
+
shape: 'exactly one human-written question per repository',
|
|
348
|
+
},
|
|
349
|
+
protocol: { entryPoint, k, repositoryScope: 'explicit', scoring: 'exact labeled file path within top-k of results from the requested repository' },
|
|
350
|
+
floor: { value: floorValue, committed: floor?.hitTop5Floor ?? null, absolute: ABSOLUTE_FLOOR, acceptedForRelease: floor?.acceptedForRelease ?? null },
|
|
351
|
+
totals,
|
|
352
|
+
// Named so no reader can mistake availability for answer accuracy.
|
|
353
|
+
meaning: {
|
|
354
|
+
repoCoverage: 'the requested repository returned at least one of its own passages. This is an AVAILABILITY check, NOT answer accuracy.',
|
|
355
|
+
hitTop5: 'the exact pre-labeled file appeared in the top 5. Equivalent content under a different filename scores as a MISS and is not given retrospective credit.',
|
|
356
|
+
notMeasured: 'generated-answer correctness and citation support were NOT evaluated. Unscoped (whole-corpus) discovery was NOT measured.',
|
|
357
|
+
c3: 'ADR-086 C3 (>=95% Hit@5 per repository on the machine-generated oracle) is NOT demonstrated by this report and was NOT met; its diagnostic result is published alongside.',
|
|
358
|
+
},
|
|
359
|
+
rows,
|
|
360
|
+
};
|
|
361
|
+
return { report, gate };
|
|
362
|
+
}
|
|
363
|
+
|
|
364
|
+
const arg = (argv, name, fallback = null) => {
|
|
365
|
+
const index = argv.indexOf(name);
|
|
366
|
+
return index >= 0 && argv[index + 1] ? argv[index + 1] : fallback;
|
|
367
|
+
};
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* Measure a SEALED archive: extract it, find the store root, and grade that — never the build
|
|
371
|
+
* directory the archive was assembled from. Returns the report bound to the archive's own digest.
|
|
372
|
+
*/
|
|
373
|
+
export async function runRepoRecallOnBundle({ bundleFile, fixtureFile, floorFile, k = DEFAULT_K } = {}) {
|
|
374
|
+
const bundle = path.resolve(bundleFile || '');
|
|
375
|
+
if (!bundle || !fs.existsSync(bundle) || !fs.statSync(bundle).isFile()) {
|
|
376
|
+
fail(`archive missing (${bundle || 'no path supplied'})`);
|
|
377
|
+
}
|
|
378
|
+
const archive = { file: path.basename(bundle), sha256: sha256File(bundle), bytes: fs.statSync(bundle).size };
|
|
379
|
+
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'repo-recall-'));
|
|
380
|
+
try {
|
|
381
|
+
try { await extractZip(bundle, tmp); }
|
|
382
|
+
catch (error) { fail(`cannot extract archive (${error.message})`); }
|
|
383
|
+
const roots = [];
|
|
384
|
+
const walk = (dir) => {
|
|
385
|
+
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
386
|
+
if (entry.name === 'ARCHIVE-MANIFEST.json') roots.push(dir);
|
|
387
|
+
else if (entry.isDirectory()) walk(path.join(dir, entry.name));
|
|
388
|
+
}
|
|
389
|
+
};
|
|
390
|
+
walk(tmp);
|
|
391
|
+
if (roots.length !== 1) fail(`expected exactly one ARCHIVE-MANIFEST.json in the archive, found ${roots.length}`);
|
|
392
|
+
return await runRepoRecall({ kbDir: roots[0], fixtureFile, floorFile, archive, k });
|
|
393
|
+
} finally {
|
|
394
|
+
fs.rmSync(tmp, { recursive: true, force: true });
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
export async function main(argv = process.argv.slice(2)) {
|
|
399
|
+
const kbDir = arg(argv, '--kb');
|
|
400
|
+
const bundleFile = arg(argv, '--bundle');
|
|
401
|
+
if (!kbDir && !bundleFile) {
|
|
402
|
+
process.stderr.write('usage: repo-recall.mjs (--bundle <archive.zip> | --kb <extracted root>) [--out <report.json>] [--fixture <file>] [--floor <file>]\n');
|
|
403
|
+
return 64;
|
|
404
|
+
}
|
|
405
|
+
const { report, gate } = bundleFile
|
|
406
|
+
? await runRepoRecallOnBundle({
|
|
407
|
+
bundleFile: path.resolve(bundleFile),
|
|
408
|
+
fixtureFile: arg(argv, '--fixture'),
|
|
409
|
+
floorFile: arg(argv, '--floor'),
|
|
410
|
+
})
|
|
411
|
+
: await runRepoRecall({
|
|
412
|
+
kbDir: path.resolve(kbDir),
|
|
413
|
+
fixtureFile: arg(argv, '--fixture'),
|
|
414
|
+
floorFile: arg(argv, '--floor'),
|
|
415
|
+
});
|
|
416
|
+
const out = arg(argv, '--out') || (bundleFile ? `${path.resolve(bundleFile)}.recall.json` : null);
|
|
417
|
+
if (out) fs.writeFileSync(path.resolve(out), `${JSON.stringify(report, null, 2)}\n`);
|
|
418
|
+
const t = report.totals;
|
|
419
|
+
process.stdout.write(`${JSON.stringify({
|
|
420
|
+
state: report.state,
|
|
421
|
+
questions: t.questions,
|
|
422
|
+
errors: t.errors,
|
|
423
|
+
repositoriesAnswering: `${t.repoCoverage}/${t.questions}`,
|
|
424
|
+
exactFileTop1: `${t.hitTop1}/${t.questions}`,
|
|
425
|
+
exactFileTop5: `${t.hitTop5}/${t.questions}`,
|
|
426
|
+
floor: report.floor.value,
|
|
427
|
+
failures: gate.failures,
|
|
428
|
+
}, null, 2)}\n`);
|
|
429
|
+
return gate.verdict === 'PASS' ? 0 : 1;
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
// Realpath both sides: argv[1] is whatever the caller typed, while Node resolves import.meta.url
|
|
433
|
+
// THROUGH symlinks. On macOS every os.tmpdir() path is symlinked, so a naive comparison makes this
|
|
434
|
+
// CLI silently no-op and exit 0 — the exact defect that made build-bundle report success with no
|
|
435
|
+
// archive on disk (measured 2026-09-14).
|
|
436
|
+
function isMain() {
|
|
437
|
+
try {
|
|
438
|
+
if (!process.argv[1]) return false;
|
|
439
|
+
return fs.realpathSync(process.argv[1]) === fs.realpathSync(fileURLToPath(import.meta.url));
|
|
440
|
+
} catch { return false; }
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
if (isMain()) {
|
|
444
|
+
main().then((code) => { process.exitCode = code; }).catch((error) => {
|
|
445
|
+
console.error(error.message);
|
|
446
|
+
process.exitCode = 1;
|
|
447
|
+
});
|
|
448
|
+
}
|