ruvnet-brain 4.3.21 → 4.3.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/bin/install.mjs +275 -60
- package/console/app.js +189 -9
- package/console/index.html +70 -24
- package/console/scope.css +137 -0
- package/console/scope.html +144 -0
- package/console/scope.js +209 -0
- package/console/style.css +26 -0
- package/console/tips.html +1 -0
- package/kb/corpus-release-identity.mjs +239 -0
- package/kb/update-storage-transaction.mjs +20 -3
- package/package.json +9 -2
- package/plugin/.claude-plugin/plugin.json +2 -2
- package/plugin/.codex-plugin/plugin.json +1 -1
- package/plugin/commands/checkpoint.md +61 -0
- package/plugin/hooks/codex-hooks.json +64 -1
- package/plugin/hooks/hook-contracts.json +299 -6
- package/plugin/hooks/hooks.json +81 -1
- package/plugin/mcp/server.mjs +23 -0
- package/plugin/scripts/advocacy-catalog.mjs +245 -0
- package/plugin/scripts/advocacy-route.mjs +460 -0
- package/plugin/scripts/continuation-gate.mjs +25 -2
- package/plugin/scripts/continuation-objective.mjs +7 -1
- package/plugin/scripts/continuity-hook-policy.mjs +190 -15
- package/plugin/scripts/coverage-integrity.mjs +7 -0
- package/plugin/scripts/gates.mjs +113 -10
- package/plugin/scripts/grounding-turn-gate.mjs +167 -0
- package/plugin/scripts/grounding-turn-mark.mjs +91 -0
- package/plugin/scripts/hook-shim.mjs +14 -0
- package/plugin/scripts/nightly-scheduler.mjs +37 -4
- package/plugin/scripts/project-progression-checkpoint.mjs +145 -0
- package/plugin/scripts/project-progression-contract.mjs +16 -0
- package/plugin/scripts/project-progression-hook.mjs +3 -0
- package/plugin/scripts/project-progression-producer.mjs +252 -0
- package/plugin/scripts/project-progression-reader.mjs +271 -0
- package/plugin/scripts/project-progression-session-start.mjs +93 -16
- package/plugin/scripts/project-progression-sources.mjs +220 -0
- package/plugin/scripts/project-progression-store.mjs +106 -13
- package/plugin/scripts/ruvnet-gate1-pattern.mjs +29 -0
- package/plugin/scripts/session-snapshot-hook.mjs +115 -7
- package/plugin/scripts/session-start-budget.mjs +59 -0
- package/plugin/scripts/session-start-core.mjs +234 -457
- package/plugin/scripts/session-start-fsutil.mjs +61 -0
- package/plugin/scripts/session-start-health.mjs +64 -0
- package/plugin/scripts/session-start-hook-description.mjs +45 -0
- package/plugin/scripts/session-start-issue-alert.mjs +77 -0
- package/plugin/scripts/session-start-repo-identity.mjs +54 -0
- package/plugin/scripts/session-start-signals.mjs +73 -0
- package/plugin/scripts/session-start-trace.mjs +86 -0
- package/plugin/scripts/session-start-update-plane.mjs +104 -0
- package/plugin/scripts/unprompted-runtime.mjs +32 -2
- package/plugin/skills/ruvnet-brain/PLAYBOOK.md +26 -2
- package/plugin/skills/ruvnet-brain/SKILL.md +67 -2
- package/scripts/adr-072-completion.mjs +1 -1
- package/scripts/agentdb-fleet-doctor.mjs +5 -1
- package/scripts/approved-runtime.mjs +197 -0
- package/scripts/brain-novice-50.mjs +16 -1
- package/scripts/brain-score.mjs +23 -5
- package/scripts/build-bundle.mjs +971 -530
- package/scripts/build-concepts.mjs +36 -116
- package/scripts/console-engine.test.mjs +8 -7
- package/scripts/console-runtime-identity.mjs +4 -0
- package/scripts/corpus-aggregates.mjs +94 -77
- package/scripts/corpus-candidate.mjs +475 -222
- package/scripts/corpus-next-seed.mjs +225 -0
- package/scripts/corpus-promotion.mjs +58 -0
- package/scripts/corpus-reconcile.mjs +411 -105
- package/scripts/doc-currency.mjs +16 -1
- package/scripts/dual-host-deliberation.mjs +25 -2
- package/scripts/dual-host-suggest.mjs +17 -1
- package/scripts/falsify.mjs +13 -3
- package/scripts/gist-receipts.mjs +482 -87
- package/scripts/github-health-watch.mjs +12 -2
- package/scripts/handoff-asset.mjs +34 -0
- package/scripts/hook-retirement-check.mjs +8 -1
- package/scripts/host-registry.mjs +1 -1
- package/scripts/ingest-gists.mjs +74 -101
- package/scripts/job-heartbeat.sh +77 -14
- package/scripts/learning-replay-execution.mjs +10 -4
- package/scripts/nightly-gists.sh +27 -13
- package/scripts/nightly-two-run-proof.mjs +1 -1
- package/scripts/nightly-watchdog.mjs +61 -4
- package/scripts/onboarding-console.mjs +364 -28
- package/scripts/oracle/produce-questions.mjs +293 -0
- package/scripts/oracle/producer-hosts.mjs +235 -0
- package/scripts/oracle/repo-recall.mjs +448 -0
- package/scripts/oracle/retrieval-accuracy.mjs +818 -0
- package/scripts/oracle/source-tree.mjs +165 -0
- package/scripts/oracle/source-units.mjs +391 -0
- package/scripts/oracle/spike-run.mjs +98 -0
- package/scripts/oracle/unit-inventory.mjs +141 -0
- package/scripts/oracle/unit-sampling.mjs +128 -0
- package/scripts/oracle/validate-labels.mjs +250 -0
- package/scripts/private-overlay.mjs +248 -0
- package/scripts/product-integrity-contract.mjs +1 -1
- package/scripts/proxy/claude-proxied.sh +6 -0
- package/scripts/proxy/proxy-revert.sh +5 -0
- package/scripts/proxy/proxy-up.sh +6 -0
- package/scripts/proxy/proxy-verify.mjs +4 -0
- package/scripts/public-inputs.mjs +409 -0
- package/scripts/public-verification-inputs.mjs +112 -26
- package/scripts/public-verification-lane.mjs +1 -1
- package/scripts/published-surface-probe.mjs +34 -4
- package/scripts/qe/card-lane-gate.mjs +16 -1
- package/scripts/qe/session-start-gate.mjs +16 -1
- package/scripts/rebuild-gists-from-receipts.mjs +58 -78
- package/scripts/record-lesson.mjs +4 -1
- package/scripts/rehearse-corpus-pipeline.mjs +994 -0
- package/scripts/release-abort-stale.mjs +5 -1
- package/scripts/release-authority.mjs +104 -12
- package/scripts/release-channel-kind.mjs +86 -0
- package/scripts/release-convergence-watchdog.mjs +7 -2
- package/scripts/release-projection.mjs +177 -72
- package/scripts/release-transaction-provider.mjs +47 -10
- package/scripts/release-transaction.mjs +40 -11
- package/scripts/release.mjs +252 -17
- package/scripts/retrieval-canary.mjs +87 -0
- package/scripts/rvf-index-audit.mjs +573 -13
- package/scripts/rvf-wire.mjs +269 -0
- package/scripts/seal-gist-receipt.mjs +65 -0
- package/scripts/selfcheck.mjs +42 -21
- package/scripts/source-coverage.mjs +253 -24
- package/scripts/status-honesty.mjs +25 -0
- package/scripts/sync-census.mjs +0 -0
- package/scripts/sync-version.mjs +2 -0
- package/scripts/trismart.mjs +42 -0
- package/scripts/updater-manifest.mjs +162 -0
- package/scripts/verify-channels.mjs +17 -5
- package/scripts/wired-check.mjs +48 -10
- package/tri-smart-skill/QUICKSTART.md +37 -0
- package/tri-smart-skill/README.md +92 -0
- package/tri-smart-skill/install.cmd +14 -0
- package/tri-smart-skill/install.command +13 -0
- package/tri-smart-skill/install.mjs +51 -0
- package/tri-smart-skill/install.sh +9 -0
- package/tri-smart-skill/tri-smart/SKILL.md +90 -0
- package/tri-smart-skill/tri-smart/evals/evals.json +25 -0
- package/tri-smart-skill/tri-smart/references/protocol.md +25 -0
- package/tri-smart-skill/tri-smart/references/provider-cli.md +18 -0
- package/tri-smart-skill/tri-smart/scripts/review.mjs +154 -0
- package/tri-smart-skill/tri-smart/scripts/setup.mjs +97 -0
- package/tri-smart-skill/tri-smart/scripts/verify-access.mjs +107 -0
- package/scripts/corpus-seed-publish.mjs +0 -110
|
@@ -1,132 +1,52 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// build-concepts.mjs —
|
|
3
|
-
// chunked into kb/concepts.passages.jsonl (+ concepts.meta.json) so `forge-big both --name concepts`
|
|
4
|
-
// can embed it into concepts.big.rvf. The cross-repo tool (forge-ask-all / search_ruvnet) then unions
|
|
5
|
-
// this prose layer with the deep-source repos at query time — making code-implemented capabilities
|
|
6
|
-
// retrievable as high-confidence PROSE without re-embedding the 40K-chunk source stores.
|
|
2
|
+
// build-concepts.mjs — thin CLI wrapper around the canonical concepts pipeline (Step 3, 2026-09-13).
|
|
7
3
|
//
|
|
8
|
-
//
|
|
9
|
-
//
|
|
4
|
+
// This used to be an INDEPENDENT concepts transformation with its own chunker, fence helpers, and
|
|
5
|
+
// card handling -- a real duplicate of corpus-aggregates.mjs's buildConceptAggregate. It now owns no
|
|
6
|
+
// chunking/fencing/card logic of its own: materializePublicInputs (scripts/public-inputs.mjs) does
|
|
7
|
+
// the ONE canonical public-prose selection (fencing private repos, resolving topic ownership), and
|
|
8
|
+
// buildConceptAggregate (scripts/corpus-aggregates.mjs) does the ONE canonical passage assembly --
|
|
9
|
+
// exactly the pattern Step 2 established for scripts/seal-gist-receipt.mjs as a thin wrapper over
|
|
10
|
+
// buildGistAggregate.
|
|
11
|
+
//
|
|
12
|
+
// Public prose is materialized into a throwaway temp directory first (never over the live kb/ in
|
|
13
|
+
// place, even though doing so would be safe -- see public-inputs.mjs's own header) so this tool's
|
|
14
|
+
// two responsibilities stay separated: SELECT (temp dir) then ASSEMBLE (writes into kb/ directly,
|
|
15
|
+
// matching this tool's historical behavior).
|
|
10
16
|
//
|
|
11
17
|
// node scripts/build-concepts.mjs → writes kb/concepts.passages.jsonl + kb/concepts.meta.json
|
|
12
18
|
import fs from 'node:fs';
|
|
19
|
+
import os from 'node:os';
|
|
13
20
|
import path from 'node:path';
|
|
14
21
|
import { fileURLToPath } from 'node:url';
|
|
15
|
-
import {
|
|
22
|
+
import { observeSourceUniverse } from './source-coverage.mjs';
|
|
23
|
+
import { materializePublicInputs } from './public-inputs.mjs';
|
|
24
|
+
import { buildConceptAggregate } from './corpus-aggregates.mjs';
|
|
16
25
|
|
|
17
26
|
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
18
27
|
const KB = path.join(ROOT, 'kb');
|
|
19
28
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
const
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
// l2-topics.<repo>.json to learn its slugs, so an L2 article whose slug belongs to a private repo is
|
|
36
|
-
// fenced out even when its repo attribution is unknown. Fail-closed: if a private topics file exists
|
|
37
|
-
// but can't be parsed, abort rather than ship.
|
|
38
|
-
const slugFence = loadPrivateSlugs(KB, PRIVATE);
|
|
39
|
-
if (!slugFence.ok) {
|
|
40
|
-
console.error(`[build-concepts] FATAL: ${slugFence.reason}. Refusing to build.`);
|
|
41
|
-
process.exit(1);
|
|
42
|
-
}
|
|
43
|
-
const PRIVATE_SLUGS = slugFence.slugs;
|
|
44
|
-
|
|
45
|
-
// Auto-discover every repo that has a primer (skip PRIVATE ones — their prose must not ship).
|
|
46
|
-
const REPOS = fs.readdirSync(KB)
|
|
47
|
-
.filter((f) => f.endsWith('-primer.md'))
|
|
48
|
-
.map((f) => f.replace(/-primer\.md$/, ''))
|
|
49
|
-
.filter((r) => !isPrivate(r))
|
|
50
|
-
.sort();
|
|
51
|
-
|
|
52
|
-
// slug -> repo, from the per-repo l2-topics files (+ ruflo defaults that predate them)
|
|
53
|
-
const slugRepo = new Map([['guidance-mechanism', 'ruflo'], ['memory-end-to-end', 'ruflo'], ['adr-coverage', 'ruflo']]);
|
|
54
|
-
for (const r of REPOS) {
|
|
55
|
-
const tf = path.join(KB, `l2-topics.${r}.json`);
|
|
56
|
-
if (fs.existsSync(tf)) for (const t of JSON.parse(fs.readFileSync(tf, 'utf8'))) slugRepo.set(t.slug, r);
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
// ~3200-char paragraph-aligned chunks
|
|
60
|
-
function chunk(text, size = 3200) {
|
|
61
|
-
const out = []; const paras = text.split(/\n\n+/); let buf = '';
|
|
62
|
-
for (const p of paras) {
|
|
63
|
-
if (buf && (buf.length + p.length + 2) > size) { out.push(buf); buf = ''; }
|
|
64
|
-
buf += (buf ? '\n\n' : '') + p;
|
|
29
|
+
export async function main() {
|
|
30
|
+
const observation = observeSourceUniverse({ owner: 'ruvnet', externalSources: [] });
|
|
31
|
+
const publicInputDir = fs.mkdtempSync(path.join(os.tmpdir(), 'build-concepts-public-inputs-'));
|
|
32
|
+
try {
|
|
33
|
+
const publicInputs = materializePublicInputs({ builderRoot: ROOT, outDir: publicInputDir });
|
|
34
|
+
const result = buildConceptAggregate({
|
|
35
|
+
publicInputDir,
|
|
36
|
+
selectionReceipt: publicInputs.selectionReceipt,
|
|
37
|
+
observationSha256: observation.observationSha256,
|
|
38
|
+
outDir: KB,
|
|
39
|
+
});
|
|
40
|
+
console.log(`concepts: ${result.passages} passages → kb/concepts.passages.jsonl`);
|
|
41
|
+
console.log('next: node kb/forge-big.mjs both --dir kb --name concepts (builds concepts.big.rvf)');
|
|
42
|
+
} finally {
|
|
43
|
+
fs.rmSync(publicInputDir, { recursive: true, force: true });
|
|
65
44
|
}
|
|
66
|
-
if (buf.trim()) out.push(buf);
|
|
67
|
-
return out.length ? out : [text];
|
|
68
45
|
}
|
|
69
46
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
const chunks = chunk(text);
|
|
75
|
-
chunks.forEach((c, i) => {
|
|
76
|
-
const sid = String(id++);
|
|
77
|
-
const p = `${repo}/${kind}/${slug}${chunks.length > 1 ? `#${i}` : ''}`;
|
|
78
|
-
passages.push({ id: sid, text: c, path: p, title });
|
|
79
|
-
entries[sid] = { path: p, kind: kind === 'PRIMER' ? 'doc' : 'doc', title, chunk: i, preview: c.slice(0, 200) };
|
|
47
|
+
if (((() => { try { return process.argv[1] && fs.realpathSync(process.argv[1]) === fs.realpathSync(fileURLToPath(import.meta.url)); } catch { return false; } })())) {
|
|
48
|
+
main().catch((error) => {
|
|
49
|
+
console.error(`[build-concepts] ${error.message}`);
|
|
50
|
+
process.exitCode = 1;
|
|
80
51
|
});
|
|
81
52
|
}
|
|
82
|
-
|
|
83
|
-
// 1) accepted L2 articles (skip kb/l2/rejected/)
|
|
84
|
-
let l2n = 0;
|
|
85
|
-
for (const f of fs.readdirSync(path.join(KB, 'l2'))) {
|
|
86
|
-
if (!f.endsWith('.md')) continue;
|
|
87
|
-
const slug = f.replace(/\.md$/, '');
|
|
88
|
-
const repo = slugRepo.get(slug) || 'ruvnet';
|
|
89
|
-
// Fence a private repo's L2 prose whether it's attributed by repo OR only recognizable by slug
|
|
90
|
-
// (QE-0011 security#1 — closes the fail-open 'ruvnet' default for unknown attribution).
|
|
91
|
-
if (shouldFenceL2({ repo, slug }, PRIVATE, PRIVATE_SLUGS)) continue;
|
|
92
|
-
const text = fs.readFileSync(path.join(KB, 'l2', f), 'utf8');
|
|
93
|
-
const title = (text.match(/^#\s+(.+)/m) || [, slug])[1];
|
|
94
|
-
add(repo, 'L2', slug, title, text); l2n++;
|
|
95
|
-
}
|
|
96
|
-
|
|
97
|
-
// 2) per-repo primers
|
|
98
|
-
let pn = 0;
|
|
99
|
-
for (const r of REPOS) {
|
|
100
|
-
const pf = path.join(KB, `${r}-primer.md`);
|
|
101
|
-
if (!fs.existsSync(pf)) { console.log(` (no primer for ${r})`); continue; }
|
|
102
|
-
add(r, 'PRIMER', `${r}-primer`, `${r} — Primer`, fs.readFileSync(pf, 'utf8')); pn++;
|
|
103
|
-
}
|
|
104
|
-
|
|
105
|
-
// 3) capability cards — short, capability-phrased, one per building block. These are the high-signal
|
|
106
|
-
// passages that let a BY-DESCRIPTION query ("what caches vector queries?") rerank to the right repo
|
|
107
|
-
// even when the user never names it. Each card is attributed to its repo via the path prefix.
|
|
108
|
-
let cn = 0;
|
|
109
|
-
const cardsFile = path.join(KB, 'capability-cards.md');
|
|
110
|
-
if (fs.existsSync(cardsFile)) {
|
|
111
|
-
const raw = fs.readFileSync(cardsFile, 'utf8');
|
|
112
|
-
for (const sec of raw.split(/^##\s+/m).slice(1)) {
|
|
113
|
-
const nl = sec.indexOf('\n');
|
|
114
|
-
if (nl < 0) continue;
|
|
115
|
-
const repo = sec.slice(0, nl).trim();
|
|
116
|
-
const body = sec.slice(nl + 1).trim();
|
|
117
|
-
if (!repo || !body) continue;
|
|
118
|
-
if (isPrivate(repo)) continue; // QE-0011 security#1: fence a private repo's capability card too
|
|
119
|
-
add(repo, 'CARD', `${repo}-card`, `${repo} — Capability`, `${repo} — ${body}`);
|
|
120
|
-
cn++;
|
|
121
|
-
}
|
|
122
|
-
}
|
|
123
|
-
|
|
124
|
-
fs.writeFileSync(path.join(KB, 'concepts.passages.jsonl'), passages.map((p) => JSON.stringify(p)).join('\n') + '\n');
|
|
125
|
-
fs.writeFileSync(path.join(KB, 'concepts.meta.json'), JSON.stringify({
|
|
126
|
-
model: 'concepts', dimensions: 0, metric: 'cosine', name: 'concepts',
|
|
127
|
-
generated: new Date().toISOString(), repo: 'ruvnet-concepts',
|
|
128
|
-
note: 'L2 synthesis + per-repo primers; embedded into concepts.big.rvf via forge-big.',
|
|
129
|
-
entries,
|
|
130
|
-
}, null, 2));
|
|
131
|
-
console.log(`concepts: ${l2n} L2 articles + ${pn} primers + ${cn} capability cards → ${passages.length} passages → kb/concepts.passages.jsonl`);
|
|
132
|
-
console.log('next: node kb/forge-big.mjs both --dir kb --name concepts (builds concepts.big.rvf)');
|
|
@@ -69,19 +69,20 @@ t('memory score: nothing probed → null (never a number we did not measure)', (
|
|
|
69
69
|
assert.equal(scoreMemoryHealth({ probes: {} }).score, null);
|
|
70
70
|
});
|
|
71
71
|
|
|
72
|
-
t('buildCapabilityRecommendations: OFF + verified command + evidence ->
|
|
72
|
+
t('buildCapabilityRecommendations: memory-distillation OFF + verified command + evidence -> NO rec while its inverse is unavailable', () => {
|
|
73
|
+
// Until 6a6ba72f (2026-09-07) this asserted ONE rec: distill-project's 2026-07-24 restore round-trip
|
|
74
|
+
// (644→648→644→648) was taken as a proven inverse. That commit emptied CAPABILITY_ELIGIBLE with the
|
|
75
|
+
// reason "a historical restore demonstration is not proof of a currently available inverse —
|
|
76
|
+
// distill-project --restore now refuses unsafe database replacement". A recommendation without a
|
|
77
|
+
// live undo is exactly what makeRecommendation exists to forbid, so the stronger invariant wins and
|
|
78
|
+
// this test was the stale side (red from 2026-09-07 until re-measured 2026-09-11).
|
|
73
79
|
const recs = buildCapabilityRecommendations({ capabilities: [{
|
|
74
80
|
key: 'memory-distillation', label: 'Memory distillation', scope: 'project', state: 'off',
|
|
75
81
|
whatItBuysYou: 'Loose notes get mined into reusable patterns.',
|
|
76
82
|
turnOn: { human: "Mine this project's stored memories into reusable patterns", cmd: 'node /x/distill-project.mjs' },
|
|
77
83
|
evidence: '10 memories stored and 80.0% embedded, but 0 have been distilled into patterns',
|
|
78
84
|
}] });
|
|
79
|
-
assert.equal(recs.length,
|
|
80
|
-
assert.equal(recs[0].id, 'enable:memory-distillation');
|
|
81
|
-
assert.equal(recs[0].scope, 'project');
|
|
82
|
-
assert.equal(recs[0].touchesMachine, false); // project scope -> never the machine-touch banner
|
|
83
|
-
assert.ok(recs[0].undo.human.length > 0); // schema gate: undo always present
|
|
84
|
-
assert.ok(recs[0].evidence[0].observed.includes('80.0% embedded'));
|
|
85
|
+
assert.equal(recs.length, 0, 'no proven inverse today => never offered, however complete the row looks');
|
|
85
86
|
});
|
|
86
87
|
t('buildCapabilityRecommendations: ON, IDLE, UNKNOWN, and ABSENT are never offered', () => {
|
|
87
88
|
for (const state of ['on', 'idle', 'unknown', 'absent']) {
|
|
@@ -44,6 +44,10 @@ export const CONSOLE_RUNTIME_SURFACE = Object.freeze([
|
|
|
44
44
|
'kb/lifecycle-evidence-retention.mjs',
|
|
45
45
|
'kb/model-requirements.mjs',
|
|
46
46
|
'kb/zip-extract.mjs',
|
|
47
|
+
// install.mjs imports this STATICALLY (corpus transport identity + approved-runtime stamping,
|
|
48
|
+
// ADR-086 step 16). A copied installer whose sibling is missing does not degrade — it throws
|
|
49
|
+
// ERR_MODULE_NOT_FOUND on import, before a single line runs.
|
|
50
|
+
'kb/corpus-release-identity.mjs',
|
|
47
51
|
'bin/install.mjs',
|
|
48
52
|
'package.json',
|
|
49
53
|
]);
|
|
@@ -1,33 +1,25 @@
|
|
|
1
1
|
// Rebuild the two source-derived public aggregate stores as one atomic artifact set.
|
|
2
2
|
|
|
3
|
-
import crypto from 'node:crypto';
|
|
4
3
|
import fs from 'node:fs';
|
|
5
4
|
import path from 'node:path';
|
|
6
5
|
import { spawnSync } from 'node:child_process';
|
|
7
6
|
import { fileURLToPath } from 'node:url';
|
|
8
7
|
import { promoteArtifactSet } from '../kb/incremental-refresh.mjs';
|
|
9
|
-
import {
|
|
8
|
+
import { buildGistAggregate } from './rebuild-gists-from-receipts.mjs';
|
|
10
9
|
import { writeRvfGeneration } from './rvf-generation.mjs';
|
|
10
|
+
import { digest, sha256File } from './coverage-integrity.mjs';
|
|
11
|
+
import { materializePublicInputs, SELECTION_RECEIPT_KIND, SELECTION_RECEIPT_SCHEMA,
|
|
12
|
+
validateSelectionReceipt } from './public-inputs.mjs';
|
|
11
13
|
|
|
12
14
|
const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
13
15
|
const HEX64 = /^[a-f0-9]{64}$/;
|
|
14
16
|
const MODEL = 'Xenova/bge-base-en-v1.5';
|
|
15
17
|
const DIMENSIONS = 768;
|
|
16
|
-
const AGGREGATES = ['ruv-gists', 'concepts'];
|
|
17
18
|
|
|
18
19
|
function fail(message) {
|
|
19
20
|
throw new Error(`[corpus-aggregates] ${message}`);
|
|
20
21
|
}
|
|
21
22
|
|
|
22
|
-
function readJson(file, label) {
|
|
23
|
-
try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
|
|
24
|
-
catch (error) { fail(`${label} is missing or unreadable (${error.message})`); }
|
|
25
|
-
}
|
|
26
|
-
|
|
27
|
-
function sha256File(file) {
|
|
28
|
-
return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
29
|
-
}
|
|
30
|
-
|
|
31
23
|
function chunks(text, size = 3200) {
|
|
32
24
|
const result = [];
|
|
33
25
|
let buffer = '';
|
|
@@ -42,46 +34,51 @@ function chunks(text, size = 3200) {
|
|
|
42
34
|
return result;
|
|
43
35
|
}
|
|
44
36
|
|
|
45
|
-
|
|
37
|
+
// buildConceptAggregate — Step 3 (2026-09-13): this used to be a SECOND, independently-fenced
|
|
38
|
+
// concepts transformation (its own privateStores/privateSlugs re-derivation from
|
|
39
|
+
// PRIVATE-STORES.json/l2-topics.*.json, duplicating scripts/build-concepts.mjs's separate copy of
|
|
40
|
+
// the same logic). Fencing now happens EXACTLY ONCE, upstream, in materializePublicInputs
|
|
41
|
+
// (scripts/public-inputs.mjs) -- `publicInputDir` is trusted to already contain ONLY public prose,
|
|
42
|
+
// so this function does no fencing of its own; it just assembles passages from what is actually
|
|
43
|
+
// there. `selectionReceipt` is the PublicInputSet's own receipt (proves what was excluded, by
|
|
44
|
+
// digest, without carrying private content) and `observationSha256` must be the corpus coverage's
|
|
45
|
+
// own observation identity -- callers assert that equality (rebuildCorpusAggregates does, below),
|
|
46
|
+
// not this pure function.
|
|
47
|
+
export function buildConceptAggregate({ publicInputDir, selectionReceipt, observationSha256, outDir,
|
|
46
48
|
now = () => new Date().toISOString() } = {}) {
|
|
47
|
-
const input = path.resolve(
|
|
49
|
+
const input = path.resolve(publicInputDir || '');
|
|
48
50
|
const output = path.resolve(outDir || '');
|
|
49
|
-
if (!HEX64.test(String(
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
if (!
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
51
|
+
if (!HEX64.test(String(observationSha256 || ''))) fail('concepts require an exact source observation');
|
|
52
|
+
// Step 5 remediation (2026-09-13): the receipt is schema 2 (byte-bound `files[]`); the exact
|
|
53
|
+
// kind/schema constants are the producer's own exports, never restated here.
|
|
54
|
+
if (!selectionReceipt || selectionReceipt.schemaVersion !== SELECTION_RECEIPT_SCHEMA
|
|
55
|
+
|| selectionReceipt.kind !== SELECTION_RECEIPT_KIND) {
|
|
56
|
+
fail('concepts require a valid public input selection receipt');
|
|
57
|
+
}
|
|
58
|
+
// P2 (Dual, 2026-09-14): "checks kind/schema only, then enumerates disk primers". Kind and schema
|
|
59
|
+
// are a label, not a proof — this function then reads whatever prose happens to be on disk under
|
|
60
|
+
// `publicInputDir` and embeds it permanently into concepts.passages.jsonl, where no later file
|
|
61
|
+
// deletion can remove it. So the receipt is VERIFIED against that exact directory first, by the
|
|
62
|
+
// producer's own fail-closed validator: every sealed file present with exact bytes, every included
|
|
63
|
+
// name backed, and NO unsealed managed prose riding along. Both callers
|
|
64
|
+
// (scripts/build-concepts.mjs and rebuildCorpusAggregates) hand in the producer's own receipt, so
|
|
65
|
+
// this is a cheap re-read, not a second policy.
|
|
66
|
+
try {
|
|
67
|
+
validateSelectionReceipt({ receipt: selectionReceipt, dir: input });
|
|
68
|
+
} catch (error) {
|
|
69
|
+
fail(`concepts refuse to read prose that its selection receipt does not prove (${error.message})`);
|
|
61
70
|
}
|
|
71
|
+
const ownership = new Map(Object.entries(selectionReceipt.ownership || {}));
|
|
62
72
|
|
|
63
73
|
const repositories = fs.readdirSync(input)
|
|
64
74
|
.filter((file) => file.endsWith('-primer.md'))
|
|
65
75
|
.map((file) => file.slice(0, -'-primer.md'.length))
|
|
66
|
-
.filter((store) => !privateStores.has(store.toLowerCase()))
|
|
67
76
|
.sort();
|
|
68
|
-
const slugRepository = new Map([
|
|
69
|
-
['guidance-mechanism', 'ruflo'], ['memory-end-to-end', 'ruflo'], ['adr-coverage', 'ruflo'],
|
|
70
|
-
]);
|
|
71
|
-
const inputFiles = new Set(['PRIVATE-STORES.json']);
|
|
72
|
-
for (const repository of repositories) {
|
|
73
|
-
const relative = `l2-topics.${repository}.json`;
|
|
74
|
-
const file = path.join(input, relative);
|
|
75
|
-
if (!fs.existsSync(file)) continue;
|
|
76
|
-
const topics = readJson(file, `${repository} topics`);
|
|
77
|
-
if (!Array.isArray(topics)) fail(`${repository} topics are malformed`);
|
|
78
|
-
inputFiles.add(relative);
|
|
79
|
-
for (const topic of topics) if (topic?.slug) slugRepository.set(String(topic.slug), repository);
|
|
80
|
-
}
|
|
81
77
|
|
|
82
78
|
const passages = [];
|
|
83
79
|
const entries = {};
|
|
84
80
|
let nextId = 0;
|
|
81
|
+
const inputFiles = new Set();
|
|
85
82
|
const add = (repository, kind, slug, title, body) => {
|
|
86
83
|
const parts = chunks(body);
|
|
87
84
|
for (const [index, text] of parts.entries()) {
|
|
@@ -93,15 +90,15 @@ export function buildConceptAggregate({ inputDir, outDir, sourceObservationSha25
|
|
|
93
90
|
};
|
|
94
91
|
|
|
95
92
|
const l2Dir = path.join(input, 'l2');
|
|
96
|
-
if (
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
93
|
+
if (fs.existsSync(l2Dir)) {
|
|
94
|
+
for (const name of fs.readdirSync(l2Dir).filter((file) => file.endsWith('.md')).sort()) {
|
|
95
|
+
const slug = name.slice(0, -3);
|
|
96
|
+
const repository = ownership.get(slug) || 'ruvnet';
|
|
97
|
+
const relative = `l2/${name}`;
|
|
98
|
+
const body = fs.readFileSync(path.join(input, relative), 'utf8');
|
|
99
|
+
inputFiles.add(relative);
|
|
100
|
+
add(repository, 'L2', slug, body.match(/^#\s+(.+)/m)?.[1] || slug, body);
|
|
101
|
+
}
|
|
105
102
|
}
|
|
106
103
|
for (const repository of repositories) {
|
|
107
104
|
const relative = `${repository}-primer.md`;
|
|
@@ -119,7 +116,7 @@ export function buildConceptAggregate({ inputDir, outDir, sourceObservationSha25
|
|
|
119
116
|
if (newline < 0) continue;
|
|
120
117
|
const repository = section.slice(0, newline).trim();
|
|
121
118
|
const body = section.slice(newline + 1).trim();
|
|
122
|
-
if (!repository || !body
|
|
119
|
+
if (!repository || !body) continue;
|
|
123
120
|
add(repository, 'CARD', `${repository}-card`, `${repository} — Capability`, `${repository} — ${body}`);
|
|
124
121
|
}
|
|
125
122
|
}
|
|
@@ -136,19 +133,19 @@ export function buildConceptAggregate({ inputDir, outDir, sourceObservationSha25
|
|
|
136
133
|
schemaVersion: 1,
|
|
137
134
|
kind: 'ruvnet-brain-derived-store-receipt',
|
|
138
135
|
store: 'concepts',
|
|
139
|
-
|
|
136
|
+
observationSha256,
|
|
137
|
+
selectionReceiptSha256: selectionReceipt.receiptSha256,
|
|
140
138
|
inputs: [...inputFiles].sort().map((relative) => ({ path: relative, sha256: sha256File(path.join(input, relative)) })),
|
|
141
|
-
passagesSha256:
|
|
139
|
+
passagesSha256: digest(passageBody),
|
|
142
140
|
};
|
|
143
141
|
fs.writeFileSync(path.join(output, 'concepts.sources.json'), `${JSON.stringify(receipt, null, 2)}\n`);
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
.sort((a, b) => String(a.store).localeCompare(String(b.store)));
|
|
142
|
+
// Rule 9: the store-classes registry is generated FRESH from the derived stores this call actually
|
|
143
|
+
// produced -- never merged with a stale checkout copy of public-store-classes.json. Concepts is
|
|
144
|
+
// the only derived store this pipeline builds today; ruv-gists is a repository/gist-class store,
|
|
145
|
+
// classified separately (scripts/build-bundle.mjs's own discovery), not through this registry.
|
|
146
|
+
const classes = { schemaVersion: 1, derived: [{ store: 'concepts', receipt: 'concepts.sources.json' }] };
|
|
150
147
|
fs.writeFileSync(path.join(output, 'public-store-classes.json'), `${JSON.stringify(classes, null, 2)}\n`);
|
|
151
|
-
return { passages: passages.length, receipt };
|
|
148
|
+
return { store: 'concepts', kind: 'derived-store-aggregate', passages: passages.length, receipt };
|
|
152
149
|
}
|
|
153
150
|
|
|
154
151
|
function defaultBuildVector({ root, assetsDir, store }) {
|
|
@@ -159,35 +156,55 @@ function defaultBuildVector({ root, assetsDir, store }) {
|
|
|
159
156
|
if (result.error || result.status !== 0) fail(`${store} vector build failed (${result.error?.message || `exit ${result.status}`})`);
|
|
160
157
|
}
|
|
161
158
|
|
|
162
|
-
|
|
163
|
-
|
|
159
|
+
// Gists and concepts are independent derived stores (concepts never reads gist content), so each is
|
|
160
|
+
// now built and promoted through its OWN atomic step rather than one shared stage+promotion:
|
|
161
|
+
// 1. buildGistAggregate owns the ENTIRE gist lifecycle -- capture, render, embed, seal, validate,
|
|
162
|
+
// and promote -- as one atomic unit (see gist-receipts.mjs). A failed gist build/embed leaves
|
|
163
|
+
// `assets` completely untouched and concepts is never attempted.
|
|
164
|
+
// 2. concepts is then rebuilt from the (now gist-updated) `assets` tree, exactly as before.
|
|
165
|
+
// This trades the OLD single joint promotion of both stores for two independently-atomic ones; each
|
|
166
|
+
// store's own artifact set can never be left half-written, which the joint promotion could not
|
|
167
|
+
// promise for gists specifically once gist capture/render/seal became a single multi-step pipeline.
|
|
168
|
+
export async function rebuildCorpusAggregates({ assetsDir, observation, coverage, root = ROOT, cache = null,
|
|
169
|
+
transport = {}, buildVector = defaultBuildVector, builderSha = null, allowNoPrivateFence = false,
|
|
170
|
+
now = () => new Date().toISOString() } = {}) {
|
|
164
171
|
const assets = path.resolve(assetsDir || '');
|
|
165
172
|
const observationSha256 = String(observation?.observationSha256 || '');
|
|
166
173
|
if (!HEX64.test(observationSha256)) fail('aggregate rebuild requires an exact source observation');
|
|
167
|
-
const sourceFile = path.join(assets, 'ruv-gists.sources.json');
|
|
168
|
-
const source = readJson(sourceFile, 'gist source receipt');
|
|
169
|
-
if (source.sourceObservationSha256 !== observationSha256) fail('gist source receipt observation differs from the stable observation');
|
|
170
174
|
if (typeof buildVector !== 'function') fail('aggregate vector builder is unavailable');
|
|
175
|
+
// Rule 8: the concepts receipt's observation identity must EXACTLY EQUAL corpus coverage's own
|
|
176
|
+
// observation identity, not merely happen to be fed the same value by an accident of call order.
|
|
177
|
+
if (!coverage || coverage.sourceObservationSha256 !== observationSha256) {
|
|
178
|
+
fail('concepts observation identity does not exactly equal the corpus coverage observation identity');
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
const gistAggregate = await buildGistAggregate({
|
|
182
|
+
observation, cache, outDir: assets, root, transport, buildVector, sourceCommit: observationSha256, now,
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
// Public prose selection (rule 1-3, 5): a fresh, positively-selected, already-fenced tree, rebuilt
|
|
186
|
+
// every round directly into `assets` -- never an overlay onto whatever a prior round left behind.
|
|
187
|
+
const publicInputs = materializePublicInputs({
|
|
188
|
+
builderRoot: root, outDir: assets, policy: { allowNoFence: allowNoPrivateFence }, builderSha, now,
|
|
189
|
+
});
|
|
171
190
|
|
|
172
191
|
const stage = fs.mkdtempSync(path.join(path.dirname(assets), '.corpus-aggregates-'));
|
|
173
192
|
try {
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
writeRvfGeneration({ dir: stage, previousDir: stage, store: 'concepts', model: MODEL,
|
|
193
|
+
buildConceptAggregate({
|
|
194
|
+
publicInputDir: assets, selectionReceipt: publicInputs.selectionReceipt,
|
|
195
|
+
observationSha256, outDir: stage, now,
|
|
196
|
+
});
|
|
197
|
+
await buildVector({ root, assetsDir: stage, store: 'concepts' });
|
|
198
|
+
writeRvfGeneration({ dir: stage, previousDir: assets, store: 'concepts', model: MODEL,
|
|
181
199
|
dimensions: DIMENSIONS, sourceCommit: observationSha256, builtUtc: now() });
|
|
182
200
|
const files = [
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
]),
|
|
187
|
-
'ruv-gists.sources.json', 'concepts.sources.json', 'public-store-classes.json', 'RVF-GENERATIONS.json',
|
|
201
|
+
'concepts.passages.jsonl', 'concepts.meta.json', 'concepts.big.rvf',
|
|
202
|
+
'concepts.big.rvf.idmap.json', 'concepts.big.rvf.embed.json',
|
|
203
|
+
'concepts.sources.json', 'public-store-classes.json', 'RVF-GENERATIONS.json',
|
|
188
204
|
];
|
|
189
205
|
promoteArtifactSet({ liveDir: assets, candidateDir: stage, files });
|
|
190
|
-
|
|
206
|
+
const rebuilt = gistAggregate.omitted ? ['concepts'] : ['concepts', 'ruv-gists'];
|
|
207
|
+
return { rebuilt: rebuilt.sort(), sourceObservationSha256: observationSha256, gistAggregate, publicInputs };
|
|
191
208
|
} finally {
|
|
192
209
|
fs.rmSync(stage, { recursive: true, force: true });
|
|
193
210
|
}
|