ruvnet-brain 4.2.2-dev → 4.3.1-dev

package/scripts/corpus-aggregates.mjs

@@ -0,0 +1,194 @@
+// Rebuild the two source-derived public aggregate stores as one atomic artifact set.
+
+import crypto from 'node:crypto';
+import fs from 'node:fs';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { fileURLToPath } from 'node:url';
+import { promoteArtifactSet } from '../kb/incremental-refresh.mjs';
+import { reconstructGists, writeReconstruction } from './rebuild-gists-from-receipts.mjs';
+import { writeRvfGeneration } from './rvf-generation.mjs';
+
+const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
+const HEX64 = /^[a-f0-9]{64}$/;
+const MODEL = 'Xenova/bge-base-en-v1.5';
+const DIMENSIONS = 768;
+const AGGREGATES = ['ruv-gists', 'concepts'];
+
+function fail(message) {
+  throw new Error(`[corpus-aggregates] ${message}`);
+}
+
+function readJson(file, label) {
+  try { return JSON.parse(fs.readFileSync(file, 'utf8')); }
+  catch (error) { fail(`${label} is missing or unreadable (${error.message})`); }
+}
+
+function sha256File(file) {
+  return crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
+}
+
+function chunks(text, size = 3200) {
+  const result = [];
+  let buffer = '';
+  for (const paragraph of String(text).split(/\n\n+/)) {
+    if (buffer && buffer.length + paragraph.length + 2 > size) {
+      result.push(buffer);
+      buffer = '';
+    }
+    buffer = buffer ? `${buffer}\n\n${paragraph}` : paragraph;
+  }
+  if (buffer.trim()) result.push(buffer);
+  return result;
+}
+
+export function buildConceptAggregate({ inputDir, outDir, sourceObservationSha256,
+  now = () => new Date().toISOString() } = {}) {
+  const input = path.resolve(inputDir || '');
+  const output = path.resolve(outDir || '');
+  if (!HEX64.test(String(sourceObservationSha256 || ''))) fail('concepts require an exact source observation');
+  const fenceFile = path.join(input, 'PRIVATE-STORES.json');
+  const fence = readJson(fenceFile, 'private fence');
+  if (!Array.isArray(fence.privateStores)) fail('private fence has no privateStores array');
+  const privateStores = new Set(fence.privateStores.map((store) => String(store).toLowerCase()));
+  const privateSlugs = new Set();
+  for (const store of privateStores) {
+    const file = path.join(input, `l2-topics.${store}.json`);
+    if (!fs.existsSync(file)) continue;
+    const topics = readJson(file, `${store} private topic fence`);
+    if (!Array.isArray(topics)) fail(`${store} private topic fence is malformed`);
+    for (const topic of topics) if (topic?.slug) privateSlugs.add(String(topic.slug));
+  }
+
+  const repositories = fs.readdirSync(input)
+    .filter((file) => file.endsWith('-primer.md'))
+    .map((file) => file.slice(0, -'-primer.md'.length))
+    .filter((store) => !privateStores.has(store.toLowerCase()))
+    .sort();
+  const slugRepository = new Map([
+    ['guidance-mechanism', 'ruflo'], ['memory-end-to-end', 'ruflo'], ['adr-coverage', 'ruflo'],
+  ]);
+  const inputFiles = new Set(['PRIVATE-STORES.json']);
+  for (const repository of repositories) {
+    const relative = `l2-topics.${repository}.json`;
+    const file = path.join(input, relative);
+    if (!fs.existsSync(file)) continue;
+    const topics = readJson(file, `${repository} topics`);
+    if (!Array.isArray(topics)) fail(`${repository} topics are malformed`);
+    inputFiles.add(relative);
+    for (const topic of topics) if (topic?.slug) slugRepository.set(String(topic.slug), repository);
+  }
+
+  const passages = [];
+  const entries = {};
+  let nextId = 0;
+  const add = (repository, kind, slug, title, body) => {
+    const parts = chunks(body);
+    for (const [index, text] of parts.entries()) {
+      const id = String(nextId++);
+      const passagePath = `${repository}/${kind}/${slug}${parts.length > 1 ? `#${index}` : ''}`;
+      passages.push({ id, text, path: passagePath, title });
+      entries[id] = { path: passagePath, kind: 'doc', title, chunk: index, preview: text.slice(0, 200) };
+    }
+  };
+
+  const l2Dir = path.join(input, 'l2');
+  if (!fs.existsSync(l2Dir) || !fs.statSync(l2Dir).isDirectory()) fail('concept L2 input directory is missing');
+  for (const name of fs.readdirSync(l2Dir).filter((file) => file.endsWith('.md')).sort()) {
+    const slug = name.slice(0, -3);
+    const repository = slugRepository.get(slug) || 'ruvnet';
+    if (privateStores.has(repository.toLowerCase()) || privateSlugs.has(slug)) continue;
+    const relative = `l2/${name}`;
+    const body = fs.readFileSync(path.join(input, relative), 'utf8');
+    inputFiles.add(relative);
+    add(repository, 'L2', slug, body.match(/^#\s+(.+)/m)?.[1] || slug, body);
+  }
+  for (const repository of repositories) {
+    const relative = `${repository}-primer.md`;
+    const body = fs.readFileSync(path.join(input, relative), 'utf8');
+    inputFiles.add(relative);
+    add(repository, 'PRIMER', `${repository}-primer`, `${repository} — Primer`, body);
+  }
+  const cardsRelative = 'capability-cards.md';
+  const cardsFile = path.join(input, cardsRelative);
+  if (fs.existsSync(cardsFile)) {
+    inputFiles.add(cardsRelative);
+    const sections = fs.readFileSync(cardsFile, 'utf8').split(/^##\s+/m).slice(1);
+    for (const section of sections) {
+      const newline = section.indexOf('\n');
+      if (newline < 0) continue;
+      const repository = section.slice(0, newline).trim();
+      const body = section.slice(newline + 1).trim();
+      if (!repository || !body || privateStores.has(repository.toLowerCase())) continue;
+      add(repository, 'CARD', `${repository}-card`, `${repository} — Capability`, `${repository} — ${body}`);
+    }
+  }
+  if (!passages.length) fail('concept aggregate produced zero public passages');
+
+  fs.mkdirSync(output, { recursive: true });
+  const passageBody = `${passages.map((row) => JSON.stringify(row)).join('\n')}\n`;
+  fs.writeFileSync(path.join(output, 'concepts.passages.jsonl'), passageBody);
+  fs.writeFileSync(path.join(output, 'concepts.meta.json'), `${JSON.stringify({
+    model: 'concepts', dimensions: 0, metric: 'cosine', name: 'concepts', generated: now(),
+    repo: 'ruvnet-concepts', note: 'Public L2 synthesis, primers, and capability cards.', entries,
+  }, null, 2)}\n`);
+  const receipt = {
+    schemaVersion: 1,
+    kind: 'ruvnet-brain-derived-store-receipt',
+    store: 'concepts',
+    sourceObservationSha256,
+    inputs: [...inputFiles].sort().map((relative) => ({ path: relative, sha256: sha256File(path.join(input, relative)) })),
+    passagesSha256: crypto.createHash('sha256').update(passageBody).digest('hex'),
+  };
+  fs.writeFileSync(path.join(output, 'concepts.sources.json'), `${JSON.stringify(receipt, null, 2)}\n`);
+  const classesFile = path.join(input, 'public-store-classes.json');
+  const classes = fs.existsSync(classesFile) ? readJson(classesFile, 'public store classes') : { schemaVersion: 1, derived: [] };
+  if (classes.schemaVersion !== 1 || !Array.isArray(classes.derived)) fail('public store classes are malformed');
+  classes.derived = [...classes.derived.filter((entry) => String(entry?.store || '').toLowerCase() !== 'concepts'),
+    { store: 'concepts', receipt: 'concepts.sources.json' }]
+    .sort((a, b) => String(a.store).localeCompare(String(b.store)));
+  fs.writeFileSync(path.join(output, 'public-store-classes.json'), `${JSON.stringify(classes, null, 2)}\n`);
+  return { passages: passages.length, receipt };
+}
+
+function defaultBuildVector({ root, assetsDir, store }) {
+  const script = path.join(path.resolve(root || ROOT), 'kb', 'forge-big.mjs');
+  const result = spawnSync(process.execPath, [script, 'both', '--dir', assetsDir, '--name', store], {
+    encoding: 'utf8', stdio: 'inherit', env: { ...process.env },
+  });
+  if (result.error || result.status !== 0) fail(`${store} vector build failed (${result.error?.message || `exit ${result.status}`})`);
+}
+
+export async function rebuildCorpusAggregates({ assetsDir, observation, root = ROOT,
+  fetchFn = globalThis.fetch, buildVector = defaultBuildVector, now = () => new Date().toISOString() } = {}) {
+  const assets = path.resolve(assetsDir || '');
+  const observationSha256 = String(observation?.observationSha256 || '');
+  if (!HEX64.test(observationSha256)) fail('aggregate rebuild requires an exact source observation');
+  const sourceFile = path.join(assets, 'ruv-gists.sources.json');
+  const source = readJson(sourceFile, 'gist source receipt');
+  if (source.sourceObservationSha256 !== observationSha256) fail('gist source receipt observation differs from the stable observation');
+  if (typeof buildVector !== 'function') fail('aggregate vector builder is unavailable');
+
+  const stage = fs.mkdtempSync(path.join(path.dirname(assets), '.corpus-aggregates-'));
+  try {
+    const gists = await reconstructGists(source, { fetchFn });
+    writeReconstruction(gists, { outDir: stage });
+    buildConceptAggregate({ inputDir: assets, outDir: stage, sourceObservationSha256: observationSha256, now });
+    for (const store of AGGREGATES) await buildVector({ root, assetsDir: stage, store });
+    writeRvfGeneration({ dir: stage, previousDir: assets, store: 'ruv-gists', model: MODEL,
+      dimensions: DIMENSIONS, sourceCommit: observationSha256, builtUtc: now() });
+    writeRvfGeneration({ dir: stage, previousDir: stage, store: 'concepts', model: MODEL,
+      dimensions: DIMENSIONS, sourceCommit: observationSha256, builtUtc: now() });
+    const files = [
+      ...AGGREGATES.flatMap((store) => [
+        `${store}.passages.jsonl`, `${store}.meta.json`, `${store}.big.rvf`,
+        `${store}.big.rvf.idmap.json`, `${store}.big.rvf.embed.json`,
+      ]),
+      'ruv-gists.sources.json', 'concepts.sources.json', 'public-store-classes.json', 'RVF-GENERATIONS.json',
+    ];
+    promoteArtifactSet({ liveDir: assets, candidateDir: stage, files });
+    return { rebuilt: [...AGGREGATES].sort(), sourceObservationSha256: observationSha256 };
+  } finally {
+    fs.rmSync(stage, { recursive: true, force: true });
+  }
+}

package/scripts/corpus-candidate.mjs

@@ -7,6 +7,8 @@ import os from 'node:os';
 import path from 'node:path';
 import { spawnSync } from 'node:child_process';
 import { fileURLToPath } from 'node:url';
+import { extractZip } from '../kb/zip-extract.mjs';
+import { validatePublicInventory } from './public-inventory.mjs';
 
 const HERE = path.dirname(fileURLToPath(import.meta.url));
 const REQUIRED_SUFFIXES = [
@@ -68,55 +70,62 @@ function policyRows(policy) {
 
 function assertPolicyCurrent(policy) {
   const rows = policyRows(policy);
-  const stale = rows.filter((row) => row?.eligible !== false && row?.status !== 'CURRENT');
+  const eligible = (row) => row?.eligible !== false && (!row?.disposition || row.disposition === 'eligible');
+  const stale = rows.filter((row) => eligible(row) && row?.status !== 'CURRENT');
   if (stale.length) fail(`eligibility policy has ${stale.length} eligible row(s) that are not CURRENT`);
   return {
-    eligibleRepoCount: rows.filter((row) => row?.eligible !== false && row?.status === 'CURRENT').length,
-    gistCount: rows.filter((row) => /gist/i.test(String(row?.kind || row?.type || ''))).length,
+    eligibleRepoCount: rows.filter((row) => eligible(row) && row?.kind === 'repository' && row?.status === 'CURRENT').length,
+    gistCount: rows.filter((row) => eligible(row) && /gist/i.test(String(row?.kind || row?.type || ''))).length,
   };
 }
 
-function archiveEntries(bundleFile) {
-  const result = spawnSync('unzip', ['-Z1', bundleFile], { encoding: 'utf8' });
-  if (result.status !== 0) fail(`cannot list archive (${result.stderr.trim() || `exit ${result.status}`})`);
-  const entries = result.stdout.split(/\r?\n/).filter(Boolean);
-  for (const entry of entries) {
-    const normalized = entry.replaceAll('\\', '/');
-    if (normalized.startsWith('/') || normalized.split('/').includes('..')) {
-      fail(`archive contains unsafe path ${JSON.stringify(entry)}`);
-    }
-  }
-  return entries;
-}
-
-function verifyArchive({ bundleFile, stores, privateStores }) {
+async function verifyArchive({ bundleFile, stores, privateStores }) {
   if (!bundleFile || !fs.existsSync(bundleFile)) fail(`bundle missing (${bundleFile || 'no path supplied'})`);
-  const entries = archiveEntries(bundleFile);
-  for (const privateStore of privateStores) {
-    const escaped = privateStore.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
-    const privateFile = new RegExp(`(?:^|/)${escaped}(?:\\.|$)`, 'i');
-    if (entries.some((entry) => privateFile.test(entry))) fail(`private store ${privateStore} is present in archive`);
-  }
-
   const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'corpus-candidate-'));
   try {
-    const extraction = spawnSync('unzip', ['-qq', bundleFile, '-d', tmp], { encoding: 'utf8' });
-    if (extraction.status !== 0) fail(`cannot extract archive (${extraction.stderr.trim() || `exit ${extraction.status}`})`);
+    let extraction;
+    try {
+      extraction = await extractZip(bundleFile, tmp);
+    } catch (error) {
+      fail(`cannot extract archive (${error.message})`);
+    }
+    const entries = extraction.entryNames;
+    for (const privateStore of privateStores) {
+      const escaped = privateStore.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
+      const privateFile = new RegExp(`(?:^|/)${escaped}(?:\\.|$)`, 'i');
+      if (entries.some((entry) => privateFile.test(entry.replaceAll('\\', '/')))) {
+        fail(`private store ${privateStore} is present in archive`);
+      }
+    }
     for (const store of stores) {
       for (const expected of store.files) {
-        const matches = entries.filter((entry) => path.posix.basename(entry) === expected.file);
+        const matches = entries.filter((entry) => path.posix.basename(entry.replaceAll('\\', '/')) === expected.file);
         if (matches.length !== 1) fail(`archive must contain exactly one ${expected.file}; found ${matches.length}`);
         const extracted = path.join(tmp, ...matches[0].replaceAll('\\', '/').split('/'));
         if (sha256File(extracted) !== expected.sha256) fail(`archive ${expected.file} differs from canonical assets`);
       }
     }
+    const selectedStores = new Map(stores.map(({ name }) => [name.toLowerCase(), name]));
+    const storeSuffixes = [...REQUIRED_SUFFIXES, ...OPTIONAL_SHIPPED_SUFFIXES]
+      .sort((left, right) => right.length - left.length);
+    const archiveFamilies = entries.map((entry) => path.posix.basename(entry.replaceAll('\\', '/')))
+      .map((name) => {
+        const suffix = storeSuffixes.find((candidate) => name.toLowerCase().endsWith(candidate));
+        return suffix ? { name, store: name.slice(0, -suffix.length) } : null;
+      }).filter(Boolean);
+    const unclassified = [...new Set(archiveFamilies.filter(({ store }) => !selectedStores.has(store.toLowerCase()))
+      .map(({ store }) => store))].sort();
+    if (unclassified.length) fail(`unclassified archive store(s): ${unclassified.join(', ')}`);
+    const aliases = archiveFamilies.filter(({ store }) => selectedStores.get(store.toLowerCase()) !== store)
+      .map(({ name }) => name).sort();
+    if (aliases.length) fail(`archive store-family alias differs by case: ${aliases.join(', ')}`);
   } finally {
     fs.rmSync(tmp, { recursive: true, force: true });
   }
   return fileIdentity(bundleFile);
 }
 
-function buildReceipt({
+async function buildReceipt({
   assetsDir,
   bundleFile,
   policyFile,
@@ -138,7 +147,6 @@ function buildReceipt({
     fail('RVF generation ledger has no stores object');
   }
   const policy = readJson(path.resolve(policyFile || ''), 'eligibility policy');
-  const counts = assertPolicyCurrent(policy);
 
   const rvfFiles = fs.readdirSync(assets).filter((name) => /^.+\.big\.rvf$/.test(name)).sort();
   const rvfByStore = new Map(rvfFiles.map((file) => [file.slice(0, -'.big.rvf'.length), file]));
@@ -160,6 +168,9 @@ function buildReceipt({
     .filter(([, owners]) => owners.length > 1)
     .map(([sha256, owners]) => ({ sha256, stores: owners.sort() }));
   if (duplicateRvfDigests.length) fail(`duplicate RVF bytes: ${duplicateRvfDigests.map((row) => row.stores.join('/')).join(', ')}`);
+  const publicInventory = validatePublicInventory({ assetsDir: assets, coverage: policy, ledger });
+  const publicSet = new Set(publicInventory.publicStores);
+  const counts = assertPolicyCurrent(policy);
 
   const missingSidecars = [];
   const stores = [];
@@ -172,7 +183,7 @@ function buildReceipt({
     if (!generation.sourceCommit || !generation.builtUtc || !generation.model || !Number.isInteger(generation.dimensions)) {
       fail(`${store}: generation receipt lacks sourceCommit, builtUtc, model, or dimensions provenance`);
     }
-    if (privateSet.has(store.toLowerCase())) continue;
+    if (privateSet.has(store.toLowerCase()) || !publicSet.has(store.toLowerCase())) continue;
     const files = [];
     for (const suffix of REQUIRED_SUFFIXES) {
       const file = path.join(assets, `${store}${suffix}`);
@@ -194,7 +205,14 @@ function buildReceipt({
   }
   if (missingSidecars.length) fail(`missing sidecars: ${missingSidecars.join(', ')}`);
   if (!stores.length) fail('zero public corpus stores remain after the private fence');
-  const archive = verifyArchive({ bundleFile: path.resolve(bundleFile || ''), stores, privateStores });
+  const finalByteFiles = stores.flatMap((store) => store.files.map((file) => ({ store: store.name, ...file })))
+    .sort((a, b) => a.file.localeCompare(b.file) || a.store.localeCompare(b.store));
+  const finalBytePartitionSha256 = crypto.createHash('sha256').update(canonicalJson({
+    publicInventoryPartitionSha256: publicInventory.partitionSha256,
+    publicStores: publicInventory.publicStores,
+    files: finalByteFiles,
+  })).digest('hex');
+  const archive = await verifyArchive({ bundleFile: path.resolve(bundleFile || ''), stores, privateStores });
   const policyIdentity = fileIdentity(path.resolve(policyFile));
   const fenceIdentity = fileIdentity(fenceFile);
   const ledgerIdentity = fileIdentity(ledgerFile);
@@ -213,6 +231,8 @@ function buildReceipt({
     privateFence: fenceIdentity,
     eligibilityPolicy: policyIdentity,
     generationLedger: ledgerIdentity,
+    publicInventory,
+    finalBytePartitionSha256,
     excludedPrivateStores: privateStores.filter((privateStore) =>
       [...rvfByStore.keys()].some((store) => store.toLowerCase() === privateStore)),
     duplicateRvfDigests,
@@ -229,9 +249,9 @@ function currentGitSha() {
   return result.stdout.trim();
 }
 
-export function createCorpusReceipt(options) {
+export async function createCorpusReceipt(options) {
   const receiptFile = path.resolve(options.receiptFile || 'dist/corpus-receipt.json');
-  const receipt = buildReceipt({
+  const receipt = await buildReceipt({
     ...options,
     builderSourceSha: options.builderSourceSha || currentGitSha(),
     createdAt: options.createdAt || new Date().toISOString(),
@@ -241,7 +261,7 @@ export function createCorpusReceipt(options) {
   return receipt;
 }
 
-export function verifyCorpusReceipt(options) {
+export async function verifyCorpusReceipt(options) {
   const receipt = readJson(path.resolve(options.receiptFile || ''), 'corpus receipt');
   if (receipt.kind !== 'ruvnet-brain-corpus-candidate' || receipt.schemaVersion !== 1) fail('unsupported corpus receipt');
   const exactFiles = [
@@ -262,7 +282,7 @@ export function verifyCorpusReceipt(options) {
     if (sha256File(file) !== expected?.sha256) fail(`${label} sha256 differs from receipt`);
     if (fs.statSync(file).size !== expected?.bytes) fail(`${label} byte length differs from receipt`);
   }
-  const actual = buildReceipt({
+  const actual = await buildReceipt({
     ...options,
     builderSourceSha: receipt.builderSourceSha,
     createdAt: receipt.createdAt,
@@ -285,7 +305,7 @@ async function main() {
     receiptFile: arg('--receipt', arg('--out', 'dist/corpus-receipt.json')),
     builderSourceSha: arg('--builder-source-sha'),
   };
-  const receipt = mode === 'verify' ? verifyCorpusReceipt(options) : createCorpusReceipt(options);
+  const receipt = mode === 'verify' ? await verifyCorpusReceipt(options) : await createCorpusReceipt(options);
   console.log(JSON.stringify({ ok: true, mode, archive: receipt.archive, stores: receipt.storeCount }, null, 2));
 }