ruvector 0.2.29 → 0.2.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/cli.js +624 -88
- package/bin/mcp-server.js +198 -17
- package/dist/core/embedding-provenance.d.ts +145 -0
- package/dist/core/embedding-provenance.d.ts.map +1 -0
- package/dist/core/embedding-provenance.js +258 -0
- package/dist/core/index.d.ts +1 -0
- package/dist/core/index.d.ts.map +1 -1
- package/dist/core/index.js +1 -0
- package/dist/core/intelligence-engine.d.ts +65 -4
- package/dist/core/intelligence-engine.d.ts.map +1 -1
- package/dist/core/intelligence-engine.js +149 -12
- package/dist/core/onnx/bundled-parallel.mjs +24 -19
- package/dist/core/onnx/loader.js +31 -4
- package/dist/core/onnx-embedder.d.ts +42 -1
- package/dist/core/onnx-embedder.d.ts.map +1 -1
- package/dist/core/onnx-embedder.js +116 -11
- package/dist/core/onnx-optimized.d.ts +8 -1
- package/dist/core/onnx-optimized.d.ts.map +1 -1
- package/dist/core/onnx-optimized.js +41 -6
- package/package.json +5 -4
package/bin/cli.js
CHANGED
|
@@ -183,42 +183,92 @@ program
|
|
|
183
183
|
.command('insert <database> <file>')
|
|
184
184
|
.description('Insert vectors from JSON file')
|
|
185
185
|
.option('-b, --batch-size <number>', 'Batch size for insertion', '1000')
|
|
186
|
-
.action((dbPath, file, options) => {
|
|
186
|
+
.action(async (dbPath, file, options) => {
|
|
187
187
|
requireRuvector();
|
|
188
188
|
const spinner = ora('Loading database...').start();
|
|
189
189
|
|
|
190
190
|
try {
|
|
191
|
-
// Read dimension from sidecar (
|
|
191
|
+
// Read dimension + embedding provenance from sidecar (#508, ADR-210 D0).
|
|
192
|
+
// Sidecar JSON is untrusted on-disk input: malformed records are treated
|
|
193
|
+
// as absent (sanitizeDimension / sanitizeProvenanceSafe), never crash.
|
|
192
194
|
let dimension = 384;
|
|
195
|
+
let storeProvenance = null;
|
|
193
196
|
const metaPath = `${dbPath}.meta.json`;
|
|
194
197
|
if (fs.existsSync(metaPath)) {
|
|
195
|
-
try {
|
|
198
|
+
try {
|
|
199
|
+
const meta = JSON.parse(fs.readFileSync(metaPath, 'utf8'));
|
|
200
|
+
dimension = sanitizeDimension(meta.dimension, 384);
|
|
201
|
+
storeProvenance = sanitizeProvenanceSafe(meta.provenance);
|
|
202
|
+
} catch (_) {}
|
|
196
203
|
}
|
|
197
204
|
|
|
198
|
-
|
|
205
|
+
spinner.text = 'Reading vectors...';
|
|
206
|
+
const data = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
207
|
+
// Accept a plain array (raw vectors, no declared provenance) or the
|
|
208
|
+
// ADR-210 object form `{ provenance, vectors }` from embedding-path
|
|
209
|
+
// exporters. A malformed declared provenance is treated as undeclared
|
|
210
|
+
// (the dimension gate below still applies).
|
|
211
|
+
let declaredProvenance = null;
|
|
212
|
+
let vectors;
|
|
213
|
+
if (Array.isArray(data)) {
|
|
214
|
+
vectors = data;
|
|
215
|
+
} else if (data && Array.isArray(data.vectors)) {
|
|
216
|
+
vectors = data.vectors;
|
|
217
|
+
declaredProvenance = sanitizeProvenanceSafe(data.provenance);
|
|
218
|
+
} else {
|
|
219
|
+
vectors = [data];
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// ADR-210 D0: a store stamped with embedding provenance refuses
|
|
223
|
+
// mismatched inserts — clear error naming both sides, no coercion.
|
|
224
|
+
if (storeProvenance) {
|
|
225
|
+
const provMod = loadProvenance();
|
|
226
|
+
const describe = provMod ? provMod.describeProvenance : (p) => JSON.stringify(p);
|
|
227
|
+
const badDim = vectors.find(v => v && Array.isArray(v.vector) && v.vector.length !== storeProvenance.dimension);
|
|
228
|
+
const provMismatch = declaredProvenance && provMod
|
|
229
|
+
? provMod.compareProvenance(storeProvenance, declaredProvenance)
|
|
230
|
+
: [];
|
|
231
|
+
if (badDim || provMismatch.length > 0) {
|
|
232
|
+
const incoming = declaredProvenance
|
|
233
|
+
? describe(declaredProvenance)
|
|
234
|
+
: `${badDim.vector.length}-dimensional vectors with undeclared provenance`;
|
|
235
|
+
spinner.fail(chalk.red(
|
|
236
|
+
`Insert refused (ADR-210): ${dbPath} records embedding provenance ${describe(storeProvenance)}, ` +
|
|
237
|
+
`but the incoming data is ${incoming}` +
|
|
238
|
+
(provMismatch.length ? ` (differs on: ${provMismatch.join(', ')})` : '') +
|
|
239
|
+
`. Mixed stores are never created — re-embed the data or the store.`
|
|
240
|
+
));
|
|
241
|
+
process.exit(1);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
199
244
|
|
|
200
|
-
|
|
201
|
-
|
|
245
|
+
// New database: derive dimension from the data and write the sidecar
|
|
246
|
+
// so later stats/search invocations open it correctly (#508). Declared
|
|
247
|
+
// provenance from the embedding path is stamped alongside (ADR-210 D0).
|
|
248
|
+
if (!fs.existsSync(dbPath) && vectors.length > 0 && Array.isArray(vectors[0].vector)) {
|
|
249
|
+
dimension = vectors[0].vector.length;
|
|
250
|
+
const meta = { dimension };
|
|
251
|
+
if (declaredProvenance) meta.provenance = declaredProvenance;
|
|
252
|
+
try { fs.writeFileSync(metaPath, JSON.stringify(meta, null, 2)); } catch (_) {}
|
|
202
253
|
}
|
|
203
254
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
const
|
|
255
|
+
// The native binding loads/persists through storagePath itself —
|
|
256
|
+
// VectorDB has no load()/save() methods (#508).
|
|
257
|
+
const db = new VectorDB({ dimensions: dimension, storagePath: dbPath });
|
|
207
258
|
|
|
208
259
|
spinner.text = `Inserting ${vectors.length} vectors...`;
|
|
209
260
|
const batchSize = parseInt(options.batchSize);
|
|
210
261
|
|
|
211
262
|
for (let i = 0; i < vectors.length; i += batchSize) {
|
|
212
263
|
const batch = vectors.slice(i, i + batchSize);
|
|
213
|
-
db.insertBatch(batch);
|
|
264
|
+
await db.insertBatch(batch);
|
|
214
265
|
spinner.text = `Inserted ${Math.min(i + batchSize, vectors.length)}/${vectors.length} vectors...`;
|
|
215
266
|
}
|
|
216
267
|
|
|
217
|
-
db.save(dbPath);
|
|
218
268
|
spinner.succeed(chalk.green(`Inserted ${vectors.length} vectors`));
|
|
219
269
|
|
|
220
|
-
const
|
|
221
|
-
console.log(chalk.gray(` Total vectors: ${
|
|
270
|
+
const count = await db.len();
|
|
271
|
+
console.log(chalk.gray(` Total vectors: ${count}`));
|
|
222
272
|
} catch (error) {
|
|
223
273
|
spinner.fail(chalk.red('Failed to insert vectors'));
|
|
224
274
|
console.error(chalk.red(error.message));
|
|
@@ -234,7 +284,7 @@ program
|
|
|
234
284
|
.option('-k, --top-k <number>', 'Number of results', '10')
|
|
235
285
|
.option('-t, --threshold <number>', 'Similarity threshold', '0.0')
|
|
236
286
|
.option('-f, --filter <json>', 'Metadata filter as JSON')
|
|
237
|
-
.action((dbPath, options) => {
|
|
287
|
+
.action(async (dbPath, options) => {
|
|
238
288
|
requireRuvector();
|
|
239
289
|
const spinner = ora('Loading database...').start();
|
|
240
290
|
|
|
@@ -243,11 +293,16 @@ program
|
|
|
243
293
|
let dimension = 384;
|
|
244
294
|
const metaPath = `${dbPath}.meta.json`;
|
|
245
295
|
if (fs.existsSync(metaPath)) {
|
|
246
|
-
try { dimension = JSON.parse(fs.readFileSync(metaPath, 'utf8')).dimension
|
|
296
|
+
try { dimension = sanitizeDimension(JSON.parse(fs.readFileSync(metaPath, 'utf8')).dimension, 384); } catch (_) {}
|
|
297
|
+
}
|
|
298
|
+
|
|
299
|
+
if (!fs.existsSync(dbPath)) {
|
|
300
|
+
spinner.fail(chalk.red(`Database not found: ${dbPath}`));
|
|
301
|
+
process.exit(1);
|
|
247
302
|
}
|
|
248
303
|
|
|
304
|
+
// storagePath loads the existing store; VectorDB has no load() (#508).
|
|
249
305
|
const db = new VectorDB({ dimensions: dimension, storagePath: dbPath });
|
|
250
|
-
db.load(dbPath);
|
|
251
306
|
|
|
252
307
|
spinner.text = 'Searching...';
|
|
253
308
|
|
|
@@ -262,7 +317,7 @@ program
|
|
|
262
317
|
query.filter = JSON.parse(options.filter);
|
|
263
318
|
}
|
|
264
319
|
|
|
265
|
-
const results = db.search(query);
|
|
320
|
+
const results = await db.search(query);
|
|
266
321
|
spinner.succeed(chalk.green(`Found ${results.length} results`));
|
|
267
322
|
|
|
268
323
|
console.log(chalk.cyan('\nSearch Results:'));
|
|
@@ -284,35 +339,40 @@ program
|
|
|
284
339
|
program
|
|
285
340
|
.command('stats <database>')
|
|
286
341
|
.description('Show database statistics')
|
|
287
|
-
.action((dbPath) => {
|
|
342
|
+
.action(async (dbPath) => {
|
|
288
343
|
requireRuvector();
|
|
289
344
|
const spinner = ora('Loading database...').start();
|
|
290
345
|
|
|
291
346
|
try {
|
|
292
|
-
// Read dimension from sidecar (avoids JSON-parsing binary redb)
|
|
347
|
+
// Read dimension/metric from sidecar (avoids JSON-parsing binary redb)
|
|
293
348
|
let dimension = 384;
|
|
349
|
+
let metric = 'cosine';
|
|
294
350
|
const metaPath = `${dbPath}.meta.json`;
|
|
295
351
|
if (fs.existsSync(metaPath)) {
|
|
296
|
-
try {
|
|
352
|
+
try {
|
|
353
|
+
const meta = JSON.parse(fs.readFileSync(metaPath, 'utf8'));
|
|
354
|
+
dimension = sanitizeDimension(meta.dimension, dimension);
|
|
355
|
+
metric = typeof meta.metric === 'string' ? meta.metric : metric;
|
|
356
|
+
} catch (_) {}
|
|
297
357
|
}
|
|
298
358
|
|
|
299
|
-
|
|
300
|
-
|
|
359
|
+
if (!fs.existsSync(dbPath)) {
|
|
360
|
+
spinner.fail(chalk.red(`Database not found: ${dbPath}`));
|
|
361
|
+
process.exit(1);
|
|
362
|
+
}
|
|
301
363
|
|
|
302
|
-
|
|
364
|
+
// storagePath loads the existing store; VectorDB exposes len(),
|
|
365
|
+
// not a stats() aggregate (#508).
|
|
366
|
+
const db = new VectorDB({ dimensions: dimension, storagePath: dbPath });
|
|
367
|
+
const count = await db.len();
|
|
303
368
|
spinner.succeed(chalk.green('Database statistics'));
|
|
304
369
|
|
|
305
370
|
console.log(chalk.cyan('\nDatabase Stats:'));
|
|
306
|
-
console.log(chalk.white(` Vector Count: ${chalk.yellow(
|
|
307
|
-
console.log(chalk.white(` Dimension: ${chalk.yellow(
|
|
308
|
-
console.log(chalk.white(` Metric: ${chalk.yellow(
|
|
371
|
+
console.log(chalk.white(` Vector Count: ${chalk.yellow(count)}`));
|
|
372
|
+
console.log(chalk.white(` Dimension: ${chalk.yellow(dimension)}`));
|
|
373
|
+
console.log(chalk.white(` Metric: ${chalk.yellow(metric)}`));
|
|
309
374
|
console.log(chalk.white(` Implementation: ${chalk.yellow(getImplementationType())}`));
|
|
310
375
|
|
|
311
|
-
if (stats.memoryUsage) {
|
|
312
|
-
const mb = (stats.memoryUsage / (1024 * 1024)).toFixed(2);
|
|
313
|
-
console.log(chalk.white(` Memory Usage: ${chalk.yellow(mb + ' MB')}`));
|
|
314
|
-
}
|
|
315
|
-
|
|
316
376
|
const fileStats = fs.statSync(dbPath);
|
|
317
377
|
const fileMb = (fileStats.size / (1024 * 1024)).toFixed(2);
|
|
318
378
|
console.log(chalk.white(` File Size: ${chalk.yellow(fileMb + ' MB')}`));
|
|
@@ -1815,6 +1875,96 @@ program
|
|
|
1815
1875
|
console.log('');
|
|
1816
1876
|
});
|
|
1817
1877
|
|
|
1878
|
+
// =============================================================================
|
|
1879
|
+
// Tiny Dancer - cost-optimal FastGRNN model router (train + route)
|
|
1880
|
+
// =============================================================================
|
|
1881
|
+
|
|
1882
|
+
const tinyDancer = program
|
|
1883
|
+
.command('tiny-dancer')
|
|
1884
|
+
.alias('td')
|
|
1885
|
+
.description('Cost-optimal FastGRNN model router — train from a DRACO dataset and route with it (requires @ruvector/tiny-dancer)');
|
|
1886
|
+
|
|
1887
|
+
function loadTinyDancer() {
|
|
1888
|
+
try {
|
|
1889
|
+
return require('@ruvector/tiny-dancer');
|
|
1890
|
+
} catch (e) {
|
|
1891
|
+
console.error(chalk.red('\n This command requires @ruvector/tiny-dancer'));
|
|
1892
|
+
console.error(chalk.yellow(' Install it: npm install @ruvector/tiny-dancer'));
|
|
1893
|
+
console.error(chalk.dim(' (native router; ships for linux/macos/windows incl. musl + arm64)\n'));
|
|
1894
|
+
process.exit(1);
|
|
1895
|
+
}
|
|
1896
|
+
}
|
|
1897
|
+
|
|
1898
|
+
tinyDancer
|
|
1899
|
+
.command('train <draco>')
|
|
1900
|
+
.description('Train a FastGRNN router from a DRACO dataset (rows of {embedding, scores}) into a .safetensors model')
|
|
1901
|
+
.requiredOption('--out <path>', 'Output .safetensors model path')
|
|
1902
|
+
.option('--input-dim <n>', 'Embedding/feature dimension (default: inferred from the first row)')
|
|
1903
|
+
.option('--prices <json>', 'Price table as JSON or @file, e.g. \'{"haiku":1,"opus":15}\'')
|
|
1904
|
+
.option('--epochs <n>', 'Training epochs', '40')
|
|
1905
|
+
.option('--lr <n>', 'Learning rate', '0.05')
|
|
1906
|
+
.option('--hidden <n>', 'Hidden dimension', '12')
|
|
1907
|
+
.option('--tolerance <n>', 'Cheap-model "good enough" tolerance', '0.05')
|
|
1908
|
+
.action(async (draco, options) => {
|
|
1909
|
+
const td = loadTinyDancer();
|
|
1910
|
+
const parsed = JSON.parse(fs.readFileSync(draco, 'utf8'));
|
|
1911
|
+
const rows = Array.isArray(parsed) ? parsed : parsed.rows;
|
|
1912
|
+
const prices = options.prices
|
|
1913
|
+
? JSON.parse(options.prices.startsWith('@') ? fs.readFileSync(options.prices.slice(1), 'utf8') : options.prices)
|
|
1914
|
+
: (parsed.prices || {});
|
|
1915
|
+
if (!Array.isArray(rows) || rows.length === 0) {
|
|
1916
|
+
console.error(chalk.red(' DRACO file must contain rows of { embedding, scores }')); process.exit(1);
|
|
1917
|
+
}
|
|
1918
|
+
if (!prices || Object.keys(prices).length === 0) {
|
|
1919
|
+
console.error(chalk.red(' Provide a price table via --prices or a "prices" field in the file')); process.exit(1);
|
|
1920
|
+
}
|
|
1921
|
+
const inputDim = options.inputDim ? parseInt(options.inputDim, 10) : (rows[0].embedding || []).length;
|
|
1922
|
+
console.log(chalk.cyan(`\n Training FastGRNN router: ${rows.length} rows, dim ${inputDim}`));
|
|
1923
|
+
const res = await td.trainRouter(rows, prices, {
|
|
1924
|
+
outputPath: options.out,
|
|
1925
|
+
inputDim,
|
|
1926
|
+
hiddenDim: parseInt(options.hidden, 10),
|
|
1927
|
+
epochs: parseInt(options.epochs, 10),
|
|
1928
|
+
learningRate: parseFloat(options.lr),
|
|
1929
|
+
tolerance: parseFloat(options.tolerance),
|
|
1930
|
+
});
|
|
1931
|
+
console.log(chalk.green(` ✓ trained: acc=${res.trainAccuracy.toFixed(3)} val=${res.valAccuracy.toFixed(3)} loss=${res.trainLoss.toFixed(4)}`));
|
|
1932
|
+
console.log(chalk.white(` ✓ saved: ${res.modelPath} (${res.modelBytes} bytes, ${res.epochsRun} epochs)`));
|
|
1933
|
+
console.log(chalk.gray(` Load it: new Router({ modelPath: '${res.modelPath}' })\n`));
|
|
1934
|
+
});
|
|
1935
|
+
|
|
1936
|
+
tinyDancer
|
|
1937
|
+
.command('score <model>')
|
|
1938
|
+
.description('Score a query embedding with a trained model. High = the cheap model is good enough (route cheap)')
|
|
1939
|
+
.requiredOption('--query <json>', 'Query embedding as a JSON array or @file (length must match the model input dim)')
|
|
1940
|
+
.option('--threshold <n>', 'Decision threshold for cheap-vs-strong', '0.5')
|
|
1941
|
+
.action(async (model, options) => {
|
|
1942
|
+
const td = loadTinyDancer();
|
|
1943
|
+
const embedding = JSON.parse(options.query.startsWith('@') ? fs.readFileSync(options.query.slice(1), 'utf8') : options.query);
|
|
1944
|
+
const s = await td.score(model, embedding);
|
|
1945
|
+
const threshold = parseFloat(options.threshold);
|
|
1946
|
+
console.log(chalk.cyan(`\n score = ${s.toFixed(4)}`));
|
|
1947
|
+
console.log(
|
|
1948
|
+
s >= threshold
|
|
1949
|
+
? chalk.green(' → route to the CHEAP model (good enough)\n')
|
|
1950
|
+
: chalk.yellow(' → route to a STRONGER model\n')
|
|
1951
|
+
);
|
|
1952
|
+
});
|
|
1953
|
+
|
|
1954
|
+
tinyDancer
|
|
1955
|
+
.command('info')
|
|
1956
|
+
.description('Show tiny-dancer availability and version')
|
|
1957
|
+
.action(() => {
|
|
1958
|
+
try {
|
|
1959
|
+
const td = require('@ruvector/tiny-dancer');
|
|
1960
|
+
console.log(chalk.green(`\n @ruvector/tiny-dancer ${td.version()} — ${td.hello()}`));
|
|
1961
|
+
console.log(chalk.gray(' train: npx ruvector tiny-dancer train <draco.json> --out model.safetensors'));
|
|
1962
|
+
console.log(chalk.gray(' score: npx ruvector tiny-dancer score <model.safetensors> --query <embedding.json>\n'));
|
|
1963
|
+
} catch {
|
|
1964
|
+
console.log(chalk.yellow('\n @ruvector/tiny-dancer not installed. npm install @ruvector/tiny-dancer\n'));
|
|
1965
|
+
}
|
|
1966
|
+
});
|
|
1967
|
+
|
|
1818
1968
|
// =============================================================================
|
|
1819
1969
|
// Server Commands - HTTP/gRPC server
|
|
1820
1970
|
// =============================================================================
|
|
@@ -1920,35 +2070,33 @@ program
|
|
|
1920
2070
|
const spinner = ora('Exporting database...').start();
|
|
1921
2071
|
|
|
1922
2072
|
try {
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
// Load database
|
|
1926
|
-
const db = new VectorDB({ dimension: 384 }); // Will be overwritten by load
|
|
1927
|
-
if (fs.existsSync(dbPath)) {
|
|
1928
|
-
db.load(dbPath);
|
|
1929
|
-
} else {
|
|
2073
|
+
if (!fs.existsSync(dbPath)) {
|
|
1930
2074
|
spinner.fail(chalk.red(`Database not found: ${dbPath}`));
|
|
1931
2075
|
process.exit(1);
|
|
1932
2076
|
}
|
|
1933
2077
|
|
|
1934
|
-
const
|
|
1935
|
-
const data = {
|
|
1936
|
-
version: packageJson.version,
|
|
1937
|
-
exportedAt: new Date().toISOString(),
|
|
1938
|
-
stats: stats,
|
|
1939
|
-
vectors: [] // Would contain actual vector data
|
|
1940
|
-
};
|
|
2078
|
+
const outputFile = options.output || `${dbPath.replace(/\/$/, '')}_export.${options.format}`;
|
|
1941
2079
|
|
|
1942
|
-
|
|
1943
|
-
|
|
1944
|
-
|
|
1945
|
-
|
|
1946
|
-
|
|
2080
|
+
// Read dimension/metric from sidecar; storagePath loads the store (#508)
|
|
2081
|
+
let dimension = 384;
|
|
2082
|
+
const metaPath = `${dbPath}.meta.json`;
|
|
2083
|
+
if (fs.existsSync(metaPath)) {
|
|
2084
|
+
try { dimension = sanitizeDimension(JSON.parse(fs.readFileSync(metaPath, 'utf8')).dimension, 384); } catch (_) {}
|
|
1947
2085
|
}
|
|
1948
|
-
|
|
1949
|
-
|
|
1950
|
-
|
|
1951
|
-
|
|
2086
|
+
const db = new VectorDB({ dimensions: dimension, storagePath: dbPath });
|
|
2087
|
+
const count = await db.len();
|
|
2088
|
+
|
|
2089
|
+
// HONESTY: VectorDB has no enumeration API, so vector payloads cannot
|
|
2090
|
+
// be exported yet — only metadata. Refuse to write a file that import
|
|
2091
|
+
// would silently pretend to restore.
|
|
2092
|
+
spinner.fail(chalk.yellow(
|
|
2093
|
+
`Export is not supported yet: the database has ${count} vectors but ` +
|
|
2094
|
+
`the VectorDB API has no enumeration method to read them back out. ` +
|
|
2095
|
+
`The .db file itself is the portable artifact — copy it (with its ` +
|
|
2096
|
+
`.meta.json sidecar) to back up or move the database.`
|
|
2097
|
+
));
|
|
2098
|
+
console.log(chalk.gray(` Requested output: ${outputFile} (not written)`));
|
|
2099
|
+
process.exit(1);
|
|
1952
2100
|
} catch (error) {
|
|
1953
2101
|
spinner.fail(chalk.red('Export failed'));
|
|
1954
2102
|
console.error(chalk.red(error.message));
|
|
@@ -1975,20 +2123,47 @@ program
|
|
|
1975
2123
|
const data = JSON.parse(fs.readFileSync(file, 'utf8'));
|
|
1976
2124
|
const dbPath = options.database || file.replace(/_export\.json$/, '');
|
|
1977
2125
|
|
|
1978
|
-
|
|
2126
|
+
// A plain JSON array of {vector, metadata} entries is importable via
|
|
2127
|
+
// the real API. The old _export.json format never contained vectors,
|
|
2128
|
+
// so importing it would fabricate an empty database (#508).
|
|
2129
|
+
const vectors = Array.isArray(data) ? data : null;
|
|
2130
|
+
if (!vectors || vectors.length === 0 || !vectors[0].vector) {
|
|
2131
|
+
spinner.fail(chalk.yellow(
|
|
2132
|
+
'Import expects a JSON array of {vector, metadata} entries ' +
|
|
2133
|
+
'(the same format `ruvector insert` accepts). Legacy _export.json ' +
|
|
2134
|
+
'files contain no vector data and cannot be restored. To move a ' +
|
|
2135
|
+
'database, copy the .db file and its .meta.json sidecar.'
|
|
2136
|
+
));
|
|
2137
|
+
process.exit(1);
|
|
2138
|
+
}
|
|
1979
2139
|
|
|
1980
|
-
|
|
1981
|
-
|
|
1982
|
-
|
|
1983
|
-
|
|
1984
|
-
}
|
|
2140
|
+
spinner.text = `Importing ${vectors.length} vectors...`;
|
|
2141
|
+
const dimension = vectors[0].vector.length;
|
|
2142
|
+
|
|
2143
|
+
// ADR-210 D0: refuse mismatched imports into a provenance-stamped store.
|
|
2144
|
+
const importMetaPath = `${dbPath}.meta.json`;
|
|
2145
|
+
if (fs.existsSync(importMetaPath)) {
|
|
2146
|
+
let targetProvenance = null;
|
|
2147
|
+
try { targetProvenance = sanitizeProvenanceSafe(JSON.parse(fs.readFileSync(importMetaPath, 'utf8')).provenance); } catch (_) {}
|
|
2148
|
+
if (targetProvenance && targetProvenance.dimension !== dimension) {
|
|
2149
|
+
const provMod = loadProvenance();
|
|
2150
|
+
const describe = provMod ? provMod.describeProvenance : (p) => JSON.stringify(p);
|
|
2151
|
+
spinner.fail(chalk.red(
|
|
2152
|
+
`Import refused (ADR-210): ${dbPath} records embedding provenance ${describe(targetProvenance)}, ` +
|
|
2153
|
+
`but the incoming data is ${dimension}-dimensional with undeclared provenance. ` +
|
|
2154
|
+
`Mixed stores are never created — re-embed the data or the store.`
|
|
2155
|
+
));
|
|
2156
|
+
process.exit(1);
|
|
2157
|
+
}
|
|
2158
|
+
}
|
|
1985
2159
|
|
|
1986
|
-
|
|
1987
|
-
db.
|
|
2160
|
+
const db = new VectorDB({ dimensions: dimension, storagePath: dbPath });
|
|
2161
|
+
await db.insertBatch(vectors);
|
|
2162
|
+
const count = await db.len();
|
|
1988
2163
|
|
|
1989
2164
|
spinner.succeed(chalk.green(`Imported to: ${dbPath}`));
|
|
1990
|
-
console.log(chalk.gray(`
|
|
1991
|
-
console.log(chalk.gray(`
|
|
2165
|
+
console.log(chalk.gray(` Vectors imported: ${vectors.length} (db total: ${count})`));
|
|
2166
|
+
console.log(chalk.gray(` Dimension: ${dimension}`));
|
|
1992
2167
|
} catch (error) {
|
|
1993
2168
|
spinner.fail(chalk.red('Import failed'));
|
|
1994
2169
|
console.error(chalk.red(error.message));
|
|
@@ -2572,7 +2747,11 @@ program
|
|
|
2572
2747
|
const spinner = ora('Creating demo database...').start();
|
|
2573
2748
|
|
|
2574
2749
|
try {
|
|
2575
|
-
|
|
2750
|
+
// Explicit path + sidecar so the stats/search/insert/export commands
|
|
2751
|
+
// can open this database afterwards with the right dimension (#508).
|
|
2752
|
+
const demoPath = './demo.db';
|
|
2753
|
+
const db = new VectorDB({ dimensions: 4, distanceMetric: 'cosine', storagePath: demoPath });
|
|
2754
|
+
fs.writeFileSync(`${demoPath}.meta.json`, JSON.stringify({ dimension: 4, metric: 'cosine' }, null, 2));
|
|
2576
2755
|
|
|
2577
2756
|
spinner.text = 'Inserting vectors...';
|
|
2578
2757
|
// VectorDBWrapper.insert takes a single object: { id?, vector, metadata? }.
|
|
@@ -2597,6 +2776,9 @@ program
|
|
|
2597
2776
|
});
|
|
2598
2777
|
|
|
2599
2778
|
console.log(chalk.green('\n Demo complete!'));
|
|
2779
|
+
console.log(chalk.cyan('\n The database persists at ./demo.db — try:'));
|
|
2780
|
+
console.log(chalk.white(' npx ruvector stats ./demo.db'));
|
|
2781
|
+
console.log(chalk.white(' npx ruvector search ./demo.db --vector "[0.8, 0.6, 0, 0]"'));
|
|
2600
2782
|
} catch (error) {
|
|
2601
2783
|
spinner.fail(chalk.red('Demo failed'));
|
|
2602
2784
|
console.error(chalk.red(error.message));
|
|
@@ -2797,6 +2979,44 @@ function loadIntelligenceEngine() {
|
|
|
2797
2979
|
return IntelligenceEngine;
|
|
2798
2980
|
}
|
|
2799
2981
|
|
|
2982
|
+
// ADR-210 D0: shared embedding-provenance invariant (compare/refuse logic,
|
|
2983
|
+
// legacy-default derivation, rollout-flag resolution). Lazy, same pattern as
|
|
2984
|
+
// the engine: when dist is missing the CLI degrades to pre-ADR-210 behavior.
|
|
2985
|
+
let provenanceMod = null;
|
|
2986
|
+
let provenanceLoadAttempted = false;
|
|
2987
|
+
function loadProvenance() {
|
|
2988
|
+
if (provenanceLoadAttempted) return provenanceMod;
|
|
2989
|
+
provenanceLoadAttempted = true;
|
|
2990
|
+
try {
|
|
2991
|
+
provenanceMod = require('../dist/core/embedding-provenance.js');
|
|
2992
|
+
} catch (e) {
|
|
2993
|
+
provenanceMod = null;
|
|
2994
|
+
}
|
|
2995
|
+
return provenanceMod;
|
|
2996
|
+
}
|
|
2997
|
+
|
|
2998
|
+
/**
|
|
2999
|
+
* Sanitize a provenance record read from disk (untrusted JSON, ADR-210
|
|
3000
|
+
* security pass): malformed records are treated as ABSENT (null), never
|
|
3001
|
+
* crash. Falls back to a minimal shape check when dist is missing.
|
|
3002
|
+
*/
|
|
3003
|
+
function sanitizeProvenanceSafe(value) {
|
|
3004
|
+
const prov = loadProvenance();
|
|
3005
|
+
if (prov && typeof prov.sanitizeProvenance === 'function') {
|
|
3006
|
+
return prov.sanitizeProvenance(value);
|
|
3007
|
+
}
|
|
3008
|
+
return (
|
|
3009
|
+
value && typeof value === 'object' && !Array.isArray(value) &&
|
|
3010
|
+
typeof value.embedderKind === 'string' &&
|
|
3011
|
+
Number.isInteger(value.dimension) && value.dimension > 0 && value.dimension <= 65536
|
|
3012
|
+
) ? value : null;
|
|
3013
|
+
}
|
|
3014
|
+
|
|
3015
|
+
/** Bound a dimension read from an untrusted sidecar to a sane integer. */
|
|
3016
|
+
function sanitizeDimension(value, fallback) {
|
|
3017
|
+
return (Number.isInteger(value) && value > 0 && value <= 65536) ? value : fallback;
|
|
3018
|
+
}
|
|
3019
|
+
|
|
2800
3020
|
class Intelligence {
|
|
2801
3021
|
constructor(options = {}) {
|
|
2802
3022
|
this.intelPath = this.getIntelPath();
|
|
@@ -2927,16 +3147,25 @@ class Intelligence {
|
|
|
2927
3147
|
try {
|
|
2928
3148
|
if (fs.existsSync(this.intelPath)) {
|
|
2929
3149
|
const data = JSON.parse(fs.readFileSync(this.intelPath, 'utf-8'));
|
|
2930
|
-
// Merge with defaults to ensure all fields exist
|
|
3150
|
+
// Merge with defaults to ensure all fields exist. The file is
|
|
3151
|
+
// untrusted on-disk input (ADR-210 security pass): shape-check each
|
|
3152
|
+
// field so a hand-edited/corrupted store cannot crash later code
|
|
3153
|
+
// that iterates arrays or spreads objects.
|
|
3154
|
+
const asArray = (v, dflt) => (Array.isArray(v) ? v : dflt);
|
|
3155
|
+
const asObject = (v, dflt) => (v && typeof v === 'object' && !Array.isArray(v) ? v : dflt);
|
|
2931
3156
|
return {
|
|
2932
|
-
patterns: data.patterns
|
|
2933
|
-
memories: data.memories
|
|
2934
|
-
trajectories: data.trajectories
|
|
2935
|
-
errors: data.errors
|
|
2936
|
-
file_sequences: data.file_sequences
|
|
2937
|
-
agents: data.agents
|
|
2938
|
-
edges: data.edges
|
|
2939
|
-
stats: { ...defaults.stats, ...(data.stats
|
|
3157
|
+
patterns: asObject(data.patterns, defaults.patterns),
|
|
3158
|
+
memories: asArray(data.memories, defaults.memories),
|
|
3159
|
+
trajectories: asArray(data.trajectories, defaults.trajectories),
|
|
3160
|
+
errors: asObject(data.errors, defaults.errors),
|
|
3161
|
+
file_sequences: asArray(data.file_sequences, defaults.file_sequences),
|
|
3162
|
+
agents: asObject(data.agents, defaults.agents),
|
|
3163
|
+
edges: asArray(data.edges, defaults.edges),
|
|
3164
|
+
stats: { ...defaults.stats, ...asObject(data.stats, {}) },
|
|
3165
|
+
// ADR-210 D0: embedding provenance of stored memory vectors
|
|
3166
|
+
// (null = legacy store, read-only for vector writes until reembed).
|
|
3167
|
+
// Malformed records are treated as absent (sanitized, never crash).
|
|
3168
|
+
embeddingProvenance: sanitizeProvenanceSafe(data.embeddingProvenance),
|
|
2940
3169
|
// Preserve in-flight trajectories so trajectory-end (run in a later
|
|
2941
3170
|
// process) can find what trajectory-begin recorded (#517)
|
|
2942
3171
|
activeTrajectories: data.activeTrajectories || {},
|
|
@@ -3009,11 +3238,145 @@ class Intelligence {
|
|
|
3009
3238
|
return normA > 0 && normB > 0 ? dot / (normA * normB) : 0;
|
|
3010
3239
|
}
|
|
3011
3240
|
|
|
3241
|
+
// ========================================================================
|
|
3242
|
+
// ADR-210 D0: embedding-provenance invariant for the intelligence store.
|
|
3243
|
+
// Every memory write records/validates { embedderKind, modelId, dimension,
|
|
3244
|
+
// normalize, prefixPolicy }; mismatched writes are refused, legacy stores
|
|
3245
|
+
// (memories without provenance) are read-only until `hooks reembed`.
|
|
3246
|
+
// ========================================================================
|
|
3247
|
+
|
|
3248
|
+
storedProvenance() { return this.data.embeddingProvenance || null; }
|
|
3249
|
+
|
|
3250
|
+
vectorMemoryCount() {
|
|
3251
|
+
return (this.data.memories || []).filter(m => Array.isArray(m.embedding) && m.embedding.length > 0).length;
|
|
3252
|
+
}
|
|
3253
|
+
|
|
3254
|
+
/** Store predates ADR-210 (has vectors but no provenance record). */
|
|
3255
|
+
isLegacyVectorStore() {
|
|
3256
|
+
return !this.storedProvenance() && this.vectorMemoryCount() > 0;
|
|
3257
|
+
}
|
|
3258
|
+
|
|
3259
|
+
/** Legacy default: hash, dimension inferred from the stored vectors. */
|
|
3260
|
+
inferredLegacyProvenance() {
|
|
3261
|
+
const prov = loadProvenance();
|
|
3262
|
+
const first = (this.data.memories || []).find(m => Array.isArray(m.embedding) && m.embedding.length > 0);
|
|
3263
|
+
const dim = first ? first.embedding.length : 256;
|
|
3264
|
+
if (prov) return prov.legacyHashProvenance(dim);
|
|
3265
|
+
return { embedderKind: 'hash', modelId: null, dimension: dim, normalize: false, prefixPolicy: 'none' };
|
|
3266
|
+
}
|
|
3267
|
+
|
|
3268
|
+
/** Provenance of an embedding produced by the wrapper's sync hash path. */
|
|
3269
|
+
syncWriteProvenance(embedding) {
|
|
3270
|
+
return { embedderKind: 'hash', modelId: null, dimension: embedding.length, normalize: true, prefixPolicy: 'none' };
|
|
3271
|
+
}
|
|
3272
|
+
|
|
3273
|
+
/**
|
|
3274
|
+
* Gate a vector write (throws on refusal). Stamps provenance on the first
|
|
3275
|
+
* write to a fresh store; refuses mismatched writes naming both sides;
|
|
3276
|
+
* legacy stores are read-only until re-embedded.
|
|
3277
|
+
*/
|
|
3278
|
+
checkVectorWrite(active) {
|
|
3279
|
+
const prov = loadProvenance();
|
|
3280
|
+
if (!prov || !active) return; // enforcement needs the dist module
|
|
3281
|
+
if (this.isLegacyVectorStore()) {
|
|
3282
|
+
const legacy = this.inferredLegacyProvenance();
|
|
3283
|
+
const err = new Error(
|
|
3284
|
+
`Vector store ${this.intelPath} predates embedding provenance (ADR-210) and is read-only for vector writes. ` +
|
|
3285
|
+
`Stored vectors are treated as ${prov.describeProvenance(legacy)}; the active embedder is ` +
|
|
3286
|
+
`${prov.describeProvenance(active)}. Run 'ruvector hooks reembed' to re-embed and unlock it.`
|
|
3287
|
+
);
|
|
3288
|
+
err.code = 'ERR_LEGACY_STORE_READONLY';
|
|
3289
|
+
throw err;
|
|
3290
|
+
}
|
|
3291
|
+
const stored = this.storedProvenance();
|
|
3292
|
+
if (!stored) {
|
|
3293
|
+
this.data.embeddingProvenance = active;
|
|
3294
|
+
return;
|
|
3295
|
+
}
|
|
3296
|
+
prov.assertProvenanceMatch(stored, active, this.intelPath);
|
|
3297
|
+
}
|
|
3298
|
+
|
|
3299
|
+
/**
|
|
3300
|
+
* Non-throwing write gate honoring RUVECTOR_REEMBED (D5):
|
|
3301
|
+
* refuse (default) → rethrow; warn → skip the write with one stderr
|
|
3302
|
+
* warning per process; auto → handled by callers that can re-embed.
|
|
3303
|
+
* Returns { ok } or { ok: false, skipped: true }.
|
|
3304
|
+
*/
|
|
3305
|
+
guardVectorWrite(active) {
|
|
3306
|
+
try {
|
|
3307
|
+
this.checkVectorWrite(active);
|
|
3308
|
+
return { ok: true };
|
|
3309
|
+
} catch (e) {
|
|
3310
|
+
const prov = loadProvenance();
|
|
3311
|
+
const policy = prov ? prov.resolveReembedPolicy() : 'refuse';
|
|
3312
|
+
if (policy === 'warn') {
|
|
3313
|
+
if (!Intelligence._reembedWarned) {
|
|
3314
|
+
Intelligence._reembedWarned = true;
|
|
3315
|
+
console.error(`ruvector: ${e.message} (RUVECTOR_REEMBED=warn: store stays read-only, write skipped)`);
|
|
3316
|
+
}
|
|
3317
|
+
return { ok: false, skipped: true, error: e.message };
|
|
3318
|
+
}
|
|
3319
|
+
// 'auto' without an async re-embed path behaves like refuse, with a hint.
|
|
3320
|
+
if (policy === 'auto') e.message += ` (RUVECTOR_REEMBED=auto: run 'ruvector hooks reembed' — in-place re-embedding needs the async path)`;
|
|
3321
|
+
throw e;
|
|
3322
|
+
}
|
|
3323
|
+
}
|
|
3324
|
+
|
|
3325
|
+
/**
|
|
3326
|
+
* Re-embed every stored memory with `embedFn` and stamp `provenance`.
|
|
3327
|
+
* Requires retained source text; memories without text must be dropped
|
|
3328
|
+
* explicitly (the command refuses otherwise — no fabricated vectors).
|
|
3329
|
+
*
|
|
3330
|
+
* ADR-210 D3: when `embedBatchFn` is provided and the store holds
|
|
3331
|
+
* `batchThreshold` (32) or more re-embeddable memories, the whole corpus
|
|
3332
|
+
* is embedded in one bulk call (the engine routes it through the bundled
|
|
3333
|
+
* parallel worker pool); smaller stores embed per-item via `embedFn`.
|
|
3334
|
+
*/
|
|
3335
|
+
async reembedAll(embedFn, provenance, { dropMissing = false, embedBatchFn = null, batchThreshold = 32 } = {}) {
|
|
3336
|
+
const memories = Array.isArray(this.data.memories) ? this.data.memories : [];
|
|
3337
|
+
const kept = [];
|
|
3338
|
+
let dropped = 0;
|
|
3339
|
+
for (const m of memories) {
|
|
3340
|
+
if (m && typeof m.content === 'string' && m.content.length > 0) {
|
|
3341
|
+
kept.push(m);
|
|
3342
|
+
} else if (dropMissing) {
|
|
3343
|
+
dropped++;
|
|
3344
|
+
} else {
|
|
3345
|
+
throw new Error('memory without retained source text encountered; rerun with --drop-missing');
|
|
3346
|
+
}
|
|
3347
|
+
}
|
|
3348
|
+
let usedBulk = false;
|
|
3349
|
+
if (embedBatchFn && kept.length >= batchThreshold) {
|
|
3350
|
+
const vectors = await embedBatchFn(kept.map(m => m.content));
|
|
3351
|
+
if (!Array.isArray(vectors) || vectors.length !== kept.length) {
|
|
3352
|
+
throw new Error(`bulk embed returned ${vectors && vectors.length} vectors for ${kept.length} texts`);
|
|
3353
|
+
}
|
|
3354
|
+
for (let i = 0; i < kept.length; i++) kept[i].embedding = vectors[i];
|
|
3355
|
+
usedBulk = true;
|
|
3356
|
+
} else {
|
|
3357
|
+
for (const m of kept) m.embedding = await embedFn(m.content);
|
|
3358
|
+
}
|
|
3359
|
+
this.data.memories = kept;
|
|
3360
|
+
this.data.stats.total_memories = kept.length;
|
|
3361
|
+
this.data.embeddingProvenance = provenance;
|
|
3362
|
+
return { reembedded: kept.length, dropped, bulk: usedBulk };
|
|
3363
|
+
}
|
|
3364
|
+
|
|
3012
3365
|
// Memory operations - use engine's VectorDB for semantic search
|
|
3013
3366
|
async rememberAsync(memoryType, content, metadata = {}) {
|
|
3014
3367
|
if (this.engine) {
|
|
3368
|
+
let entry = null;
|
|
3015
3369
|
try {
|
|
3016
|
-
|
|
3370
|
+
entry = await this.engine.remember(content, memoryType);
|
|
3371
|
+
} catch {}
|
|
3372
|
+
if (entry) {
|
|
3373
|
+
// ADR-210 D0: validate provenance BEFORE persisting; provenance
|
|
3374
|
+
// refusals propagate (no silent fallback into a mixed store).
|
|
3375
|
+
const active = typeof this.engine.getActiveProvenance === 'function'
|
|
3376
|
+
? this.engine.getActiveProvenance()
|
|
3377
|
+
: this.syncWriteProvenance(entry.embedding);
|
|
3378
|
+
const guard = this.guardVectorWrite(active);
|
|
3379
|
+
if (!guard.ok) return null;
|
|
3017
3380
|
// Also store in legacy format for compatibility
|
|
3018
3381
|
this.data.memories.push({
|
|
3019
3382
|
id: entry.id,
|
|
@@ -3026,7 +3389,7 @@ class Intelligence {
|
|
|
3026
3389
|
if (this.data.memories.length > 5000) this.data.memories.splice(0, 1000);
|
|
3027
3390
|
this.data.stats.total_memories = this.data.memories.length;
|
|
3028
3391
|
return entry.id;
|
|
3029
|
-
}
|
|
3392
|
+
}
|
|
3030
3393
|
}
|
|
3031
3394
|
return this.remember(memoryType, content, metadata);
|
|
3032
3395
|
}
|
|
@@ -3034,6 +3397,10 @@ class Intelligence {
|
|
|
3034
3397
|
remember(memoryType, content, metadata = {}) {
|
|
3035
3398
|
const id = `mem_${this.now()}`;
|
|
3036
3399
|
const embedding = this.embed(content);
|
|
3400
|
+
// ADR-210 D0: refuse mismatched/legacy vector writes (throws), or skip
|
|
3401
|
+
// under RUVECTOR_REEMBED=warn (returns null).
|
|
3402
|
+
const guard = this.guardVectorWrite(this.syncWriteProvenance(embedding));
|
|
3403
|
+
if (!guard.ok) return null;
|
|
3037
3404
|
this.data.memories.push({ id, memory_type: memoryType, content, embedding, metadata, timestamp: this.now() });
|
|
3038
3405
|
if (this.data.memories.length > 5000) this.data.memories.splice(0, 1000);
|
|
3039
3406
|
this.data.stats.total_memories = this.data.memories.length;
|
|
@@ -3047,10 +3414,52 @@ class Intelligence {
|
|
|
3047
3414
|
return id;
|
|
3048
3415
|
}
|
|
3049
3416
|
|
|
3417
|
+
/**
|
|
3418
|
+
* Best-effort remember for ambient learning hooks (post-edit/post-command):
|
|
3419
|
+
* a provenance refusal must not fail the hook — note it once and move on.
|
|
3420
|
+
*/
|
|
3421
|
+
tryRemember(memoryType, content, metadata = {}) {
|
|
3422
|
+
try {
|
|
3423
|
+
return this.remember(memoryType, content, metadata);
|
|
3424
|
+
} catch (e) {
|
|
3425
|
+
if (!Intelligence._rememberSkipNoted) {
|
|
3426
|
+
Intelligence._rememberSkipNoted = true;
|
|
3427
|
+
console.error(chalk.dim(` (memory write skipped: ${e.message})`));
|
|
3428
|
+
}
|
|
3429
|
+
return null;
|
|
3430
|
+
}
|
|
3431
|
+
}
|
|
3432
|
+
|
|
3433
|
+
/**
|
|
3434
|
+
* ADR-210: reads stay allowed on legacy/mismatched stores, but similarity
|
|
3435
|
+
* against differently-embedded vectors is meaningless — say so once.
|
|
3436
|
+
*/
|
|
3437
|
+
warnRecallProvenance(active) {
|
|
3438
|
+
const prov = loadProvenance();
|
|
3439
|
+
if (!prov || !active || Intelligence._recallWarned) return;
|
|
3440
|
+
let stored = this.storedProvenance();
|
|
3441
|
+
if (!stored && this.isLegacyVectorStore()) stored = this.inferredLegacyProvenance();
|
|
3442
|
+
if (!stored) return;
|
|
3443
|
+
const mismatches = prov.compareProvenance(stored, active);
|
|
3444
|
+
if (mismatches.length > 0) {
|
|
3445
|
+
Intelligence._recallWarned = true;
|
|
3446
|
+
console.error(
|
|
3447
|
+
`ruvector: recall quality degraded — stored vectors are ${prov.describeProvenance(stored)} ` +
|
|
3448
|
+
`but the query was embedded as ${prov.describeProvenance(active)} (differs on: ${mismatches.join(', ')}). ` +
|
|
3449
|
+
`Run 'ruvector hooks reembed' to fix.`
|
|
3450
|
+
);
|
|
3451
|
+
}
|
|
3452
|
+
}
|
|
3453
|
+
|
|
3050
3454
|
async recallAsync(query, topK = 5) {
|
|
3051
3455
|
if (this.engine) {
|
|
3052
3456
|
try {
|
|
3053
3457
|
const results = await this.engine.recall(query, topK);
|
|
3458
|
+
// After recall: embedAsync has settled, so getActiveProvenance() now
|
|
3459
|
+
// reflects the embedder that actually served the query.
|
|
3460
|
+
if (typeof this.engine.getActiveProvenance === 'function') {
|
|
3461
|
+
this.warnRecallProvenance(this.engine.getActiveProvenance());
|
|
3462
|
+
}
|
|
3054
3463
|
// Return same format as sync recall() - direct memory objects
|
|
3055
3464
|
return results.map(r => ({
|
|
3056
3465
|
id: r.id,
|
|
@@ -3066,6 +3475,7 @@ class Intelligence {
|
|
|
3066
3475
|
|
|
3067
3476
|
recall(query, topK) {
|
|
3068
3477
|
const queryEmbed = this.embed(query);
|
|
3478
|
+
this.warnRecallProvenance(this.syncWriteProvenance(queryEmbed));
|
|
3069
3479
|
return this.data.memories
|
|
3070
3480
|
.map(m => ({ score: this.similarity(queryEmbed, m.embedding), memory: m }))
|
|
3071
3481
|
.sort((a, b) => b.score - a.score).slice(0, topK).map(r => r.memory);
|
|
@@ -3341,6 +3751,8 @@ class Intelligence {
|
|
|
3341
3751
|
sonaEnabled: engineStats.sonaEnabled,
|
|
3342
3752
|
attentionEnabled: engineStats.attentionEnabled,
|
|
3343
3753
|
embeddingDim: engineStats.memoryDimensions,
|
|
3754
|
+
// ADR-210 D1: which embedder actually serves embeds right now
|
|
3755
|
+
embedderKind: engineStats.embedderKind,
|
|
3344
3756
|
totalMemories: engineStats.totalMemories,
|
|
3345
3757
|
totalEpisodes: engineStats.totalEpisodes,
|
|
3346
3758
|
trajectoriesRecorded: engineStats.trajectoriesRecorded,
|
|
@@ -4238,7 +4650,8 @@ hooksCmd.command('post-edit').description('Post-edit learning').argument('<file>
|
|
|
4238
4650
|
const lastFile = intel.getLastEditedFile();
|
|
4239
4651
|
if (lastFile && lastFile !== file) intel.recordFileSequence(lastFile, file);
|
|
4240
4652
|
intel.learn(state, success ? 'successful-edit' : 'failed-edit', success ? 'completed' : 'failed', success ? 1.0 : -0.5);
|
|
4241
|
-
|
|
4653
|
+
// Best-effort: a provenance-locked store (ADR-210) must not fail the hook
|
|
4654
|
+
intel.tryRemember('edit', `${success ? 'successful' : 'failed'} edit of ${ext} in ${crate}`);
|
|
4242
4655
|
intel.save();
|
|
4243
4656
|
console.log(`📊 Learning recorded: ${success ? '✅' : '❌'} ${path.basename(file)}`);
|
|
4244
4657
|
const test = intel.shouldTest(file);
|
|
@@ -4263,7 +4676,8 @@ hooksCmd.command('post-command').description('Post-command learning').argument('
|
|
|
4263
4676
|
const success = opts.error ? false : (opts.success ?? true);
|
|
4264
4677
|
const classification = intel.classifyCommand(cmd);
|
|
4265
4678
|
intel.learn(`cmd_${classification.category}_${classification.subcategory}`, success ? 'success' : 'failure', success ? 'completed' : 'failed', success ? 0.8 : -0.3);
|
|
4266
|
-
|
|
4679
|
+
// Best-effort: a provenance-locked store (ADR-210) must not fail the hook
|
|
4680
|
+
intel.tryRemember('command', `${cmd} ${success ? 'succeeded' : 'failed'}`);
|
|
4267
4681
|
intel.save();
|
|
4268
4682
|
console.log(`📊 Command ${success ? '✅' : '❌'} recorded`);
|
|
4269
4683
|
});
|
|
@@ -4282,16 +4696,31 @@ hooksCmd.command('suggest-context').description('Suggest relevant context').acti
|
|
|
4282
4696
|
|
|
4283
4697
|
hooksCmd.command('remember').description('Store in memory').requiredOption('-t, --type <type>', 'Memory type').option('--silent', 'Suppress output').option('--semantic', 'Use ONNX semantic embeddings (slower, better quality)').argument('<content...>', 'Content').action(async (content, opts) => {
|
|
4284
4698
|
const intel = new Intelligence();
|
|
4285
|
-
|
|
4286
|
-
|
|
4287
|
-
|
|
4288
|
-
|
|
4289
|
-
|
|
4290
|
-
|
|
4291
|
-
|
|
4292
|
-
|
|
4293
|
-
|
|
4294
|
-
|
|
4699
|
+
try {
|
|
4700
|
+
let id;
|
|
4701
|
+
if (opts.semantic) {
|
|
4702
|
+
// Use async ONNX embedding
|
|
4703
|
+
id = await intel.rememberAsync(opts.type, content.join(' '));
|
|
4704
|
+
} else {
|
|
4705
|
+
id = intel.remember(opts.type, content.join(' '));
|
|
4706
|
+
}
|
|
4707
|
+
if (id === null) {
|
|
4708
|
+
// RUVECTOR_REEMBED=warn: store is read-only, write skipped (ADR-210)
|
|
4709
|
+
if (!opts.silent) {
|
|
4710
|
+
console.log(JSON.stringify({ success: false, skipped: true, reason: 'store is read-only for vector writes (embedding provenance, ADR-210); run `ruvector hooks reembed`' }));
|
|
4711
|
+
}
|
|
4712
|
+
return;
|
|
4713
|
+
}
|
|
4714
|
+
intel.save();
|
|
4715
|
+
if (!opts.silent) {
|
|
4716
|
+
console.log(JSON.stringify({ success: true, id, semantic: !!opts.semantic }));
|
|
4717
|
+
}
|
|
4718
|
+
} catch (e) {
|
|
4719
|
+
// ADR-210 D0: mismatched/legacy vector writes are refused, not coerced.
|
|
4720
|
+
if (!opts.silent) {
|
|
4721
|
+
console.log(JSON.stringify({ success: false, error: e.message, code: e.code || 'ERR_EMBEDDING_PROVENANCE' }));
|
|
4722
|
+
}
|
|
4723
|
+
process.exitCode = 1;
|
|
4295
4724
|
}
|
|
4296
4725
|
});
|
|
4297
4726
|
|
|
@@ -4306,6 +4735,113 @@ hooksCmd.command('recall').description('Search memory').argument('<query...>', '
|
|
|
4306
4735
|
console.log(JSON.stringify({ query: query.join(' '), semantic: !!opts.semantic, results: results.map(r => ({ type: r.memory_type || 'unknown', content: (r.content || '').slice(0, 200), timestamp: r.timestamp || '', score: r.score })) }, null, 2));
|
|
4307
4736
|
});
|
|
4308
4737
|
|
|
4738
|
+
// ADR-210 D1: maintenance command — re-embed hash-era memories with the
|
|
4739
|
+
// active embedder and stamp embedding provenance, unlocking legacy stores.
|
|
4740
|
+
// Possible because hook memories retain their source text (`content`).
|
|
4741
|
+
hooksCmd.command('reembed')
|
|
4742
|
+
.description('Re-embed stored memories with the active embedder and stamp embedding provenance (ADR-210)')
|
|
4743
|
+
.option('--dry-run', 'Report what would change without writing')
|
|
4744
|
+
.option('--drop-missing', 'Drop memories that no longer retain source text')
|
|
4745
|
+
.action(async (opts) => {
|
|
4746
|
+
const provMod = loadProvenance();
|
|
4747
|
+
if (!provMod) {
|
|
4748
|
+
console.log(JSON.stringify({ success: false, error: 'embedding-provenance module unavailable (dist not built)' }));
|
|
4749
|
+
process.exitCode = 1;
|
|
4750
|
+
return;
|
|
4751
|
+
}
|
|
4752
|
+
const intel = new Intelligence({ skipEngine: true }); // embedder chosen explicitly below
|
|
4753
|
+
const memories = Array.isArray(intel.data.memories) ? intel.data.memories : [];
|
|
4754
|
+
const missing = memories.filter(m => !(m && typeof m.content === 'string' && m.content.length > 0)).length;
|
|
4755
|
+
|
|
4756
|
+
if (missing > 0 && !opts.dropMissing) {
|
|
4757
|
+
// Honest refusal: those vectors cannot be re-embedded (no source text),
|
|
4758
|
+
// and keeping them would recreate a mixed store.
|
|
4759
|
+
console.log(JSON.stringify({
|
|
4760
|
+
success: false,
|
|
4761
|
+
error: `${missing} of ${memories.length} memories have no retained source text and cannot be re-embedded`,
|
|
4762
|
+
hint: 'rerun with --drop-missing to discard them, or leave the store read-only for vector writes',
|
|
4763
|
+
}));
|
|
4764
|
+
process.exitCode = 1;
|
|
4765
|
+
return;
|
|
4766
|
+
}
|
|
4767
|
+
|
|
4768
|
+
// Pick the target embedder per RUVECTOR_EMBEDDER (D5).
|
|
4769
|
+
const selection = provMod.resolveEmbedderSelection();
|
|
4770
|
+
let embedFn;
|
|
4771
|
+
let embedBatchFn = null;
|
|
4772
|
+
let shutdownPool = null;
|
|
4773
|
+
let provenance;
|
|
4774
|
+
if (selection === 'hash') {
|
|
4775
|
+
// Deterministic, offline-safe: the wrapper's own hash embedder.
|
|
4776
|
+
embedFn = async (t) => intel.embed(t);
|
|
4777
|
+
provenance = { embedderKind: 'hash', modelId: null, dimension: intel.embed('probe').length, normalize: true, prefixPolicy: 'none' };
|
|
4778
|
+
} else {
|
|
4779
|
+
const EngineClass = loadIntelligenceEngine();
|
|
4780
|
+
if (!EngineClass) {
|
|
4781
|
+
console.log(JSON.stringify({ success: false, error: 'IntelligenceEngine unavailable (dist not built); cannot re-embed semantically' }));
|
|
4782
|
+
process.exitCode = 1;
|
|
4783
|
+
return;
|
|
4784
|
+
}
|
|
4785
|
+
let engine;
|
|
4786
|
+
try {
|
|
4787
|
+
engine = new EngineClass({ enableSona: false, enableAttention: false });
|
|
4788
|
+
} catch (e) {
|
|
4789
|
+
console.log(JSON.stringify({ success: false, error: e.message }));
|
|
4790
|
+
process.exitCode = 1;
|
|
4791
|
+
return;
|
|
4792
|
+
}
|
|
4793
|
+
const ready = typeof engine.awaitOnnx === 'function' ? await engine.awaitOnnx() : false;
|
|
4794
|
+
if (!ready) {
|
|
4795
|
+
// Honest failure: re-embedding with a fallback hash would defeat the
|
|
4796
|
+
// point. Tell the operator what to do instead of fabricating quality.
|
|
4797
|
+
console.log(JSON.stringify({
|
|
4798
|
+
success: false,
|
|
4799
|
+
error: `ONNX model could not be loaded (${engine.getOnnxInitError?.()?.message || 'offline?'}); semantic re-embedding is impossible right now`,
|
|
4800
|
+
hint: 'retry with network access, or force the hash embedder with RUVECTOR_EMBEDDER=hash',
|
|
4801
|
+
}));
|
|
4802
|
+
process.exitCode = 1;
|
|
4803
|
+
return;
|
|
4804
|
+
}
|
|
4805
|
+
embedFn = (t) => engine.embedAsync(t);
|
|
4806
|
+
// ADR-210 D3: 32+ memories re-embed in one bulk call through the
|
|
4807
|
+
// bundled parallel worker pool (parallel-fp32; see embedBulk for the
|
|
4808
|
+
// int8 status). The pool's worker threads keep the process alive, so
|
|
4809
|
+
// they are shut down once the bulk work completes.
|
|
4810
|
+
if (typeof engine.embedBatchAsync === 'function') {
|
|
4811
|
+
embedBatchFn = (texts) => engine.embedBatchAsync(texts);
|
|
4812
|
+
shutdownPool = () => (typeof engine.shutdownEmbedderPool === 'function' ? engine.shutdownEmbedderPool() : Promise.resolve());
|
|
4813
|
+
}
|
|
4814
|
+
provenance = engine.getActiveProvenance();
|
|
4815
|
+
}
|
|
4816
|
+
|
|
4817
|
+
if (opts.dryRun) {
|
|
4818
|
+
console.log(JSON.stringify({
|
|
4819
|
+
success: true,
|
|
4820
|
+
dryRun: true,
|
|
4821
|
+
wouldReembed: memories.length - missing,
|
|
4822
|
+
wouldDrop: opts.dropMissing ? missing : 0,
|
|
4823
|
+
targetProvenance: provenance,
|
|
4824
|
+
}));
|
|
4825
|
+
return;
|
|
4826
|
+
}
|
|
4827
|
+
|
|
4828
|
+
try {
|
|
4829
|
+
const startMs = Date.now();
|
|
4830
|
+
const result = await intel.reembedAll(embedFn, provenance, { dropMissing: !!opts.dropMissing, embedBatchFn });
|
|
4831
|
+
intel.save();
|
|
4832
|
+
let parallelWorkers = 0;
|
|
4833
|
+
try {
|
|
4834
|
+
parallelWorkers = require('../dist/core/onnx-embedder.js').getParallelWorkerCount();
|
|
4835
|
+
} catch (_) {}
|
|
4836
|
+
console.log(JSON.stringify({ success: true, ...result, parallelWorkers, elapsedMs: Date.now() - startMs, provenance }));
|
|
4837
|
+
} catch (e) {
|
|
4838
|
+
console.log(JSON.stringify({ success: false, error: e.message }));
|
|
4839
|
+
process.exitCode = 1;
|
|
4840
|
+
} finally {
|
|
4841
|
+
if (shutdownPool) await shutdownPool().catch(() => {});
|
|
4842
|
+
}
|
|
4843
|
+
});
|
|
4844
|
+
|
|
4309
4845
|
hooksCmd.command('pre-compact').description('Pre-compact hook').option('--auto', 'Auto mode').action(() => {
|
|
4310
4846
|
const intel = new Intelligence();
|
|
4311
4847
|
intel.save();
|