renor-engine 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -661
- package/NOTICE +8 -6
- package/README.md +6 -6
- package/dist/.tsbuildinfo +1 -0
- package/dist/BUILD.json +2 -2
- package/dist/api/server.js +10 -8
- package/dist/buildIdentity.js +18 -1
- package/dist/callPhases.js +82 -0
- package/dist/cli.js +10 -1
- package/dist/context/rowStore.js +1 -1
- package/dist/conversation/parser.js +40 -36
- package/dist/db/client.js +6 -4
- package/dist/db/handoffTable.js +31 -0
- package/dist/db/indexSalvage.js +24 -4
- package/dist/db/migrations.js +30 -2
- package/dist/db/sqlite.js +396 -0
- package/dist/digests/digestCache.js +6 -2
- package/dist/digests/digests.js +3 -3
- package/dist/digests/worker.js +3 -1
- package/dist/embeddings/caching.js +5 -5
- package/dist/embeddings/local.js +5 -3
- package/dist/embeddings/openai.js +4 -3
- package/dist/embeddings/simdScan.js +3 -1
- package/dist/embeddings/vectorIndex.js +0 -1
- package/dist/embeddings/vertex.js +123 -13
- package/dist/eval/cacheReport.js +7 -0
- package/dist/eval/context700-candidate.js +5 -5
- package/dist/eval/evalCache.js +46 -3
- package/dist/eval/indexSnapshots.js +548 -0
- package/dist/eval/pageDump.js +67 -0
- package/dist/eval/runManifest.js +6 -0
- package/dist/eval/textTokens.js +17 -0
- package/dist/evidence/attach.js +0 -3
- package/dist/evidence/evidenceCache.js +3 -3
- package/dist/evidence/options.js +46 -25
- package/dist/evidence/secondPass.js +8 -170
- package/dist/extraction/factExtractor.js +1 -1
- package/dist/extraction/preferenceExtractor.js +1 -4
- package/dist/extraction/typedFacts.js +1 -1
- package/dist/importing/apply.js +42 -14
- package/dist/importing/cli.js +9 -8
- package/dist/importing/existing.js +2 -0
- package/dist/importing/facts.js +0 -2
- package/dist/importing/plan.js +0 -5
- package/dist/importing/source.js +1 -1
- package/dist/indexer/parser.js +4 -1
- package/dist/indexer/pipeline.js +99 -279
- package/dist/indexer/walker.js +11 -25
- package/dist/mcp/server.js +36 -6
- package/dist/memory/handoff.js +363 -0
- package/dist/memory/lifecycle.js +310 -75
- package/dist/memory/sourceName.js +77 -1
- package/dist/memory/statusFacts.js +61 -0
- package/dist/memorymap/memoryMap.js +39 -13
- package/dist/memorymap/replacements.js +8 -3
- package/dist/mvp/agentMemory.js +106 -50
- package/dist/mvp/answerCheck.js +211 -0
- package/dist/mvp/cli.js +3 -1
- package/dist/mvp/compileCache.js +3 -10
- package/dist/mvp/compileSeed.js +9 -3
- package/dist/mvp/contextDistinct.js +53 -0
- package/dist/mvp/contextProfile.js +87 -0
- package/dist/mvp/directorySurface.js +68 -0
- package/dist/mvp/engine.js +39 -26
- package/dist/mvp/exportAll.js +17 -2
- package/dist/mvp/factMarkers.js +18 -2
- package/dist/mvp/history.js +5 -4
- package/dist/mvp/log.js +118 -0
- package/dist/mvp/server.js +642 -127
- package/dist/mvp/serverCard.js +12 -3
- package/dist/mvp/telemetry.js +35 -2
- package/dist/net/poolFetch.js +174 -0
- package/dist/recall/actionRecall.js +282 -0
- package/dist/recall/autoRecall.js +447 -0
- package/dist/recall/searchRecall.js +189 -0
- package/dist/retrieval/entityLookup.js +53 -4
- package/dist/retrieval/orchestrator.js +93 -8
- package/dist/retrieval/plainWording.js +236 -0
- package/dist/retrieval/readerLayout.js +143 -224
- package/dist/router/routerCache.js +2 -2
- package/dist/watcher/watcher.js +5 -3
- package/package.json +5 -7
- package/dist/importing/index.js +0 -29
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.Statement = exports.MIN_NODE = void 0;
|
|
7
|
+
exports.nodeSupportProblem = nodeSupportProblem;
|
|
8
|
+
exports.sqliteRuntimeProblem = sqliteRuntimeProblem;
|
|
9
|
+
/**
|
|
10
|
+
* The engine's SQLite driver: Node's built-in `node:sqlite`, behind the part of the better-sqlite3 API the engine uses.
|
|
11
|
+
*
|
|
12
|
+
* Why not better-sqlite3: it is a native addon that builds itself in an npm install script. `npx -y renor-engine` then
|
|
13
|
+
* breaks for anyone whose npm does not run dependency scripts (`ignore-scripts=true`, newer npm defaults), and on a Node
|
|
14
|
+
* with no prebuilt binary the install falls back to compiling C++ for over a minute while the MCP client gives up. Node
|
|
15
|
+
* ships SQLite itself (FTS5 included) and loads sqlite-vec, whose prebuilt binary is a plain optional dependency, so the
|
|
16
|
+
* engine installs with no script at all.
|
|
17
|
+
*
|
|
18
|
+
* Behaviour kept from better-sqlite3, because the engine and its tests rely on it:
|
|
19
|
+
* - rows are plain objects and BLOBs come back as Buffer; `pluck()`, `raw()` and `safeIntegers()` work as before;
|
|
20
|
+
* - `transaction(fn)` begins (or, nested, takes a savepoint), commits, and rolls back on a throw;
|
|
21
|
+
* - errors carry the SQLite code name (`SQLITE_CONSTRAINT_UNIQUE`, `SQLITE_CORRUPT`, ...) in `code`;
|
|
22
|
+
* - a 5 s busy timeout, a 16 MB page cache, and `synchronous = NORMAL` on a connection that opens a database already in
|
|
23
|
+
* WAL mode (better-sqlite3's compiled defaults; see matchWalSynchronous);
|
|
24
|
+
* Extension loading is allowed through the C API only (`loadExtension`, for sqlite-vec); SQL's `load_extension()` stays
|
|
25
|
+
* refused, as it was.
|
|
26
|
+
*/
|
|
27
|
+
const fs_1 = __importDefault(require("fs"));
|
|
28
|
+
/** The first Node with every `node:sqlite` feature used here (extension loading, `isTransaction`, `setReturnArrays`, `backup`). */
|
|
29
|
+
exports.MIN_NODE = '22.16.0';
|
|
30
|
+
/** Why this Node cannot run the engine, or null when it can. */
|
|
31
|
+
function nodeSupportProblem(version = process.versions.node) {
|
|
32
|
+
const [major, minor] = version.split('.').map(Number);
|
|
33
|
+
if (major >= 24 || (major === 22 && minor >= 16))
|
|
34
|
+
return null;
|
|
35
|
+
return `Renor needs Node ${exports.MIN_NODE} or newer (it uses Node's built-in SQLite); this is Node ${version}.`;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Why the engine cannot store anything here, as one line with its fix, or null when it can: this Node has no usable
|
|
39
|
+
* built-in SQLite, or sqlite-vec's prebuilt binary for this platform is missing or does not load. The CLI says this and
|
|
40
|
+
* exits before it starts, so an MCP client shows the reason instead of a connection that silently failed.
|
|
41
|
+
*/
|
|
42
|
+
function sqliteRuntimeProblem() {
|
|
43
|
+
const node = nodeSupportProblem();
|
|
44
|
+
if (node)
|
|
45
|
+
return `${node} Fix: install Node 22 LTS or newer (https://nodejs.org), then run the command again.`;
|
|
46
|
+
try {
|
|
47
|
+
const db = new SqliteDatabase(':memory:');
|
|
48
|
+
try {
|
|
49
|
+
require('sqlite-vec').load(db);
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
db.close();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
const reason = (err instanceof Error ? err.message.split('\n')[0] : String(err)).replace(/\.?\s*$/, '.');
|
|
57
|
+
return `the vector search extension (sqlite-vec) did not load on ${process.platform}-${process.arch}: ${reason} `
|
|
58
|
+
+ 'Fix: let npm install optional dependencies (remove omit=optional or --no-optional from your npm settings), '
|
|
59
|
+
+ 'then run `npx -y renor-engine@latest mcp` again.';
|
|
60
|
+
}
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
let sqlite;
|
|
64
|
+
function nodeSqlite() {
|
|
65
|
+
if (sqlite)
|
|
66
|
+
return sqlite;
|
|
67
|
+
const problem = nodeSupportProblem();
|
|
68
|
+
if (problem)
|
|
69
|
+
throw new Error(`${problem} Install Node 22 LTS or newer from https://nodejs.org and run the command again.`);
|
|
70
|
+
// Node 22 prints "SQLite is an experimental feature" once on load; to an MCP client that is noise on stderr, not news.
|
|
71
|
+
const emitWarning = process.emitWarning;
|
|
72
|
+
process.emitWarning = ((warning, ...rest) => {
|
|
73
|
+
if (String(warning instanceof Error ? warning.message : warning).includes('SQLite is an experimental feature'))
|
|
74
|
+
return;
|
|
75
|
+
return emitWarning.call(process, warning, ...rest);
|
|
76
|
+
});
|
|
77
|
+
try {
|
|
78
|
+
sqlite = require('node:sqlite');
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
process.emitWarning = emitWarning;
|
|
82
|
+
}
|
|
83
|
+
return sqlite;
|
|
84
|
+
}
|
|
85
|
+
/** SQLite's primary result codes, by number. */
|
|
86
|
+
const PRIMARY_CODES = ('OK ERROR INTERNAL PERM ABORT BUSY LOCKED NOMEM READONLY INTERRUPT IOERR CORRUPT NOTFOUND FULL CANTOPEN PROTOCOL EMPTY '
|
|
87
|
+
+ 'SCHEMA TOOBIG CONSTRAINT MISMATCH MISUSE NOLFS AUTH FORMAT RANGE NOTADB NOTICE WARNING').split(' ');
|
|
88
|
+
/** The extended result codes worth telling apart; any other extended code is named by its primary code. */
|
|
89
|
+
const EXTENDED_CODES = new Map(('261 BUSY_RECOVERY,517 BUSY_SNAPSHOT,773 BUSY_TIMEOUT,262 LOCKED_SHAREDCACHE,267 CORRUPT_VTAB,523 CORRUPT_SEQUENCE,'
|
|
90
|
+
+ '779 CORRUPT_INDEX,264 READONLY_RECOVERY,520 READONLY_CANTLOCK,776 READONLY_ROLLBACK,1032 READONLY_DBMOVED,'
|
|
91
|
+
+ '275 CONSTRAINT_CHECK,531 CONSTRAINT_COMMITHOOK,787 CONSTRAINT_FOREIGNKEY,1043 CONSTRAINT_FUNCTION,'
|
|
92
|
+
+ '1299 CONSTRAINT_NOTNULL,1555 CONSTRAINT_PRIMARYKEY,1811 CONSTRAINT_TRIGGER,2067 CONSTRAINT_UNIQUE,'
|
|
93
|
+
+ '2323 CONSTRAINT_VTAB,2579 CONSTRAINT_ROWID,3091 CONSTRAINT_DATATYPE').split(',').map(pair => [Number(pair.split(' ')[0]), pair.split(' ')[1]]));
|
|
94
|
+
/** Gives a `node:sqlite` error better-sqlite3's `code` (`SQLITE_<NAME>`); every other error passes through unchanged. */
|
|
95
|
+
function withSqliteCode(err) {
|
|
96
|
+
const e = err;
|
|
97
|
+
if (e && e.code === 'ERR_SQLITE_ERROR' && typeof e.errcode === 'number') {
|
|
98
|
+
const name = EXTENDED_CODES.get(e.errcode) ?? PRIMARY_CODES[e.errcode & 0xff] ?? 'ERROR';
|
|
99
|
+
try {
|
|
100
|
+
e.code = `SQLITE_${name}`;
|
|
101
|
+
}
|
|
102
|
+
catch { /* a frozen error keeps its code */ }
|
|
103
|
+
}
|
|
104
|
+
return err;
|
|
105
|
+
}
|
|
106
|
+
function guarded(work) {
|
|
107
|
+
try {
|
|
108
|
+
return work();
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
throw withSqliteCode(err);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/** Every bigint in a row (or rows) as the nearest double, as better-sqlite3 read an integer beyond 2^53. */
|
|
115
|
+
function roundBigInts(value) {
|
|
116
|
+
if (typeof value === 'bigint')
|
|
117
|
+
return Number(value);
|
|
118
|
+
if (Array.isArray(value))
|
|
119
|
+
return value.map(roundBigInts);
|
|
120
|
+
if (value !== null && typeof value === 'object' && !ArrayBuffer.isView(value)) {
|
|
121
|
+
const out = {};
|
|
122
|
+
for (const key in value)
|
|
123
|
+
out[key] = roundBigInts(value[key]);
|
|
124
|
+
return out;
|
|
125
|
+
}
|
|
126
|
+
return value;
|
|
127
|
+
}
|
|
128
|
+
function asBuffer(value) {
|
|
129
|
+
return value instanceof Uint8Array && !Buffer.isBuffer(value) ? Buffer.from(value.buffer, value.byteOffset, value.byteLength) : value;
|
|
130
|
+
}
|
|
131
|
+
/** better-sqlite3 takes named parameters as an object in any position and positional ones also as arrays. */
|
|
132
|
+
function bindArgs(params) {
|
|
133
|
+
let named;
|
|
134
|
+
const positional = [];
|
|
135
|
+
for (const p of params) {
|
|
136
|
+
if (Array.isArray(p))
|
|
137
|
+
positional.push(...p);
|
|
138
|
+
else if (p !== null && typeof p === 'object' && !ArrayBuffer.isView(p))
|
|
139
|
+
named = p;
|
|
140
|
+
else
|
|
141
|
+
positional.push(p);
|
|
142
|
+
}
|
|
143
|
+
return named === undefined ? positional : [named, ...positional];
|
|
144
|
+
}
|
|
145
|
+
/** SQL that can change a table's columns, after which this connection's prepared statements are prepared again. */
|
|
146
|
+
const SCHEMA_CHANGE = /\b(ALTER|CREATE|DROP)\b/i;
|
|
147
|
+
const WRITE_OPCODES = new Set(['JournalMode', 'Vacuum', 'VCreate', 'VDestroy', 'VUpdate', 'ParseSchema']);
|
|
148
|
+
class Statement {
|
|
149
|
+
database;
|
|
150
|
+
stmt;
|
|
151
|
+
source;
|
|
152
|
+
mode = 'object';
|
|
153
|
+
bigInts = false;
|
|
154
|
+
changesSchema;
|
|
155
|
+
generation;
|
|
156
|
+
constructor(database, stmt, source) {
|
|
157
|
+
this.database = database;
|
|
158
|
+
this.stmt = stmt;
|
|
159
|
+
this.source = source;
|
|
160
|
+
this.changesSchema = SCHEMA_CHANGE.test(source);
|
|
161
|
+
this.generation = database.schemaGeneration;
|
|
162
|
+
}
|
|
163
|
+
run(...params) {
|
|
164
|
+
const result = guarded(() => this.current().run(...bindArgs(params)));
|
|
165
|
+
if (this.changesSchema)
|
|
166
|
+
this.database.schemaChanged();
|
|
167
|
+
// `changes` is a number even under safeIntegers(), as in better-sqlite3; only lastInsertRowid becomes a bigint.
|
|
168
|
+
return { changes: Number(result.changes), lastInsertRowid: result.lastInsertRowid };
|
|
169
|
+
}
|
|
170
|
+
get(...params) {
|
|
171
|
+
const row = this.lossy(() => this.current().get(...bindArgs(params)));
|
|
172
|
+
return row === undefined ? undefined : this.shape(row);
|
|
173
|
+
}
|
|
174
|
+
all(...params) {
|
|
175
|
+
return this.lossy(() => this.current().all(...bindArgs(params))).map(row => this.shape(row));
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Errors raised while stepping get the same `SQLITE_*` code as the other calls. An integer beyond 2^53 still throws
|
|
179
|
+
* RangeError here (better-sqlite3 returned it rounded); use safeIntegers(), or get()/all(), which round as before.
|
|
180
|
+
*/
|
|
181
|
+
*iterate(...params) {
|
|
182
|
+
const rows = guarded(() => this.current().iterate(...bindArgs(params)));
|
|
183
|
+
try {
|
|
184
|
+
for (;;) {
|
|
185
|
+
const step = guarded(() => rows.next());
|
|
186
|
+
if (step.done)
|
|
187
|
+
return;
|
|
188
|
+
yield this.shape(step.value);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
finally {
|
|
192
|
+
rows.return?.();
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
pluck(toggleState = true) {
|
|
196
|
+
return this.setMode(toggleState ? 'pluck' : this.mode === 'pluck' ? 'object' : this.mode);
|
|
197
|
+
}
|
|
198
|
+
raw(toggleState = true) {
|
|
199
|
+
return this.setMode(toggleState ? 'raw' : this.mode === 'raw' ? 'object' : this.mode);
|
|
200
|
+
}
|
|
201
|
+
safeIntegers(toggleState = true) {
|
|
202
|
+
this.bigInts = toggleState;
|
|
203
|
+
this.stmt.setReadBigInts(toggleState);
|
|
204
|
+
return this;
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Whether the statement writes nothing, as SQLite's sqlite3_stmt_readonly() decides it: its program opens no write
|
|
208
|
+
* transaction and changes no journal mode, virtual table or file. Read from the statement's EXPLAIN on demand.
|
|
209
|
+
*/
|
|
210
|
+
get readonly() {
|
|
211
|
+
const program = guarded(() => this.database.prepareNative(`EXPLAIN ${this.source}`).all());
|
|
212
|
+
return !program.some(op => (op.opcode === 'Transaction' && op.p2 > 0) || WRITE_OPCODES.has(op.opcode));
|
|
213
|
+
}
|
|
214
|
+
columns() {
|
|
215
|
+
return this.current().columns();
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* better-sqlite3 returned an integer beyond 2^53 rounded to the nearest double; node:sqlite throws RangeError
|
|
219
|
+
* (ERR_OUT_OF_RANGE) instead. A statement that writes nothing is read again as bigints and rounded the same way; a
|
|
220
|
+
* statement that writes is never run twice, so its RangeError stands.
|
|
221
|
+
*/
|
|
222
|
+
lossy(read) {
|
|
223
|
+
try {
|
|
224
|
+
return guarded(read);
|
|
225
|
+
}
|
|
226
|
+
catch (err) {
|
|
227
|
+
if (this.bigInts || err.code !== 'ERR_OUT_OF_RANGE' || !this.readonly)
|
|
228
|
+
throw err;
|
|
229
|
+
this.stmt.setReadBigInts(true);
|
|
230
|
+
try {
|
|
231
|
+
return roundBigInts(guarded(read));
|
|
232
|
+
}
|
|
233
|
+
finally {
|
|
234
|
+
this.stmt.setReadBigInts(false);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* The native statement, prepared again after this connection changed the schema. SQLite re-prepares a statement by
|
|
240
|
+
* itself, but `node:sqlite` reads the result's columns before that happens, so the first run after an ALTER TABLE
|
|
241
|
+
* would return the old columns; better-sqlite3 did not.
|
|
242
|
+
*/
|
|
243
|
+
current() {
|
|
244
|
+
if (this.generation !== this.database.schemaGeneration) {
|
|
245
|
+
this.stmt = this.database.prepareNative(this.source);
|
|
246
|
+
this.stmt.setReturnArrays(this.mode !== 'object');
|
|
247
|
+
this.stmt.setReadBigInts(this.bigInts);
|
|
248
|
+
this.generation = this.database.schemaGeneration;
|
|
249
|
+
}
|
|
250
|
+
return this.stmt;
|
|
251
|
+
}
|
|
252
|
+
setMode(mode) {
|
|
253
|
+
this.mode = mode;
|
|
254
|
+
this.stmt.setReturnArrays(mode !== 'object');
|
|
255
|
+
return this;
|
|
256
|
+
}
|
|
257
|
+
shape(row) {
|
|
258
|
+
if (this.mode === 'pluck')
|
|
259
|
+
return asBuffer(row[0]);
|
|
260
|
+
if (this.mode === 'raw')
|
|
261
|
+
return row.map(asBuffer);
|
|
262
|
+
const out = {};
|
|
263
|
+
for (const key in row)
|
|
264
|
+
out[key] = asBuffer(row[key]);
|
|
265
|
+
return out;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
exports.Statement = Statement;
|
|
269
|
+
class SqliteDatabase {
|
|
270
|
+
name;
|
|
271
|
+
readonly;
|
|
272
|
+
memory;
|
|
273
|
+
db;
|
|
274
|
+
closed = false;
|
|
275
|
+
extensions = new Set();
|
|
276
|
+
/** @internal Bumped whenever this connection may have changed the schema; see Statement's `current()`. */
|
|
277
|
+
schemaGeneration = 0;
|
|
278
|
+
constructor(filename = '', options = {}) {
|
|
279
|
+
this.name = filename;
|
|
280
|
+
this.readonly = options.readonly === true;
|
|
281
|
+
this.memory = filename === ':memory:' || filename === '';
|
|
282
|
+
if (options.fileMustExist && !this.memory && !fs_1.default.existsSync(filename)) {
|
|
283
|
+
throw Object.assign(new Error(`unable to open database file: ${filename}`), { code: 'SQLITE_CANTOPEN' });
|
|
284
|
+
}
|
|
285
|
+
const { DatabaseSync } = nodeSqlite();
|
|
286
|
+
this.db = guarded(() => new DatabaseSync(filename, { readOnly: this.readonly, allowExtension: true }));
|
|
287
|
+
try {
|
|
288
|
+
this.db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.trunc(options.timeout ?? 5000))}`);
|
|
289
|
+
this.db.exec('PRAGMA cache_size = -16000');
|
|
290
|
+
this.matchWalSynchronous();
|
|
291
|
+
}
|
|
292
|
+
catch (err) {
|
|
293
|
+
this.db.close();
|
|
294
|
+
throw withSqliteCode(err);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
get open() {
|
|
298
|
+
return !this.closed;
|
|
299
|
+
}
|
|
300
|
+
get inTransaction() {
|
|
301
|
+
return !this.closed && this.db.isTransaction;
|
|
302
|
+
}
|
|
303
|
+
prepare(source) {
|
|
304
|
+
return new Statement(this, guarded(() => this.db.prepare(source)), source);
|
|
305
|
+
}
|
|
306
|
+
/** @internal The `node:sqlite` statement itself, for Statement's own bookkeeping. */
|
|
307
|
+
prepareNative(source) {
|
|
308
|
+
return this.db.prepare(source);
|
|
309
|
+
}
|
|
310
|
+
exec(source) {
|
|
311
|
+
guarded(() => this.db.exec(source));
|
|
312
|
+
if (SCHEMA_CHANGE.test(source))
|
|
313
|
+
this.schemaChanged();
|
|
314
|
+
return this;
|
|
315
|
+
}
|
|
316
|
+
/** @internal */
|
|
317
|
+
schemaChanged() {
|
|
318
|
+
this.schemaGeneration++;
|
|
319
|
+
}
|
|
320
|
+
pragma(source, options = {}) {
|
|
321
|
+
const stmt = guarded(() => this.db.prepare(`PRAGMA ${source}`));
|
|
322
|
+
if (options.simple) {
|
|
323
|
+
stmt.setReturnArrays(true);
|
|
324
|
+
const row = guarded(() => stmt.get());
|
|
325
|
+
return row === undefined ? undefined : asBuffer(row[0]);
|
|
326
|
+
}
|
|
327
|
+
return guarded(() => stmt.all()).map(row => ({ ...row }));
|
|
328
|
+
}
|
|
329
|
+
transaction(fn) {
|
|
330
|
+
const wrap = (begin) => {
|
|
331
|
+
const db = this;
|
|
332
|
+
return function sqliteTransaction(...args) {
|
|
333
|
+
const nested = db.inTransaction;
|
|
334
|
+
db.exec(nested ? 'SAVEPOINT "_renor_tx"' : begin);
|
|
335
|
+
try {
|
|
336
|
+
const result = fn.apply(this, args);
|
|
337
|
+
if (result && typeof result.then === 'function') {
|
|
338
|
+
throw new TypeError('Transaction function cannot return a promise');
|
|
339
|
+
}
|
|
340
|
+
db.exec(nested ? 'RELEASE "_renor_tx"' : 'COMMIT');
|
|
341
|
+
return result;
|
|
342
|
+
}
|
|
343
|
+
catch (err) {
|
|
344
|
+
if (db.inTransaction) {
|
|
345
|
+
if (nested)
|
|
346
|
+
db.exec('ROLLBACK TO "_renor_tx"; RELEASE "_renor_tx"');
|
|
347
|
+
else
|
|
348
|
+
db.exec('ROLLBACK');
|
|
349
|
+
}
|
|
350
|
+
throw err;
|
|
351
|
+
}
|
|
352
|
+
};
|
|
353
|
+
};
|
|
354
|
+
const tx = wrap('BEGIN');
|
|
355
|
+
tx.default = tx;
|
|
356
|
+
tx.deferred = wrap('BEGIN DEFERRED');
|
|
357
|
+
tx.immediate = wrap('BEGIN IMMEDIATE');
|
|
358
|
+
tx.exclusive = wrap('BEGIN EXCLUSIVE');
|
|
359
|
+
return tx;
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Loads an extension once per connection: a second load only registers the same functions again, and that fails
|
|
363
|
+
* ("error during initialization") inside a transaction that has deleted rows behind a virtual table, where
|
|
364
|
+
* src/indexer/outsideRoot.ts loads sqlite-vec.
|
|
365
|
+
*/
|
|
366
|
+
loadExtension(path) {
|
|
367
|
+
if (this.extensions.has(path))
|
|
368
|
+
return this;
|
|
369
|
+
guarded(() => this.db.loadExtension(path));
|
|
370
|
+
this.extensions.add(path);
|
|
371
|
+
return this;
|
|
372
|
+
}
|
|
373
|
+
backup(destinationFile) {
|
|
374
|
+
return nodeSqlite().backup(this.db, destinationFile);
|
|
375
|
+
}
|
|
376
|
+
close() {
|
|
377
|
+
if (!this.closed) {
|
|
378
|
+
this.closed = true;
|
|
379
|
+
this.db.close();
|
|
380
|
+
}
|
|
381
|
+
return this;
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* better-sqlite3 was built with SQLITE_DEFAULT_WAL_SYNCHRONOUS=1, Node's SQLite with 2. SQLite applies that default
|
|
385
|
+
* only to a connection that opens a database already in WAL mode, read-only or not: such a connection gets
|
|
386
|
+
* synchronous = NORMAL (1) here as it did there. A connection that switches a database to WAL itself (a fresh index
|
|
387
|
+
* in openDb) keeps FULL (2), as it did there.
|
|
388
|
+
*/
|
|
389
|
+
matchWalSynchronous() {
|
|
390
|
+
const mode = this.db.prepare('PRAGMA journal_mode').get();
|
|
391
|
+
if (mode?.journal_mode === 'wal')
|
|
392
|
+
this.db.exec('PRAGMA synchronous = NORMAL');
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
const Database = SqliteDatabase;
|
|
396
|
+
exports.default = Database;
|
|
@@ -7,7 +7,7 @@ exports.DigestCache = exports.DIGEST_CACHE_SCHEMA_VERSION = void 0;
|
|
|
7
7
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
8
8
|
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
9
9
|
const node_path_1 = __importDefault(require("node:path"));
|
|
10
|
-
const
|
|
10
|
+
const sqlite_1 = __importDefault(require("../db/sqlite"));
|
|
11
11
|
/**
|
|
12
12
|
* Content-addressed store for per-note digests (RENOR_DIGESTS, src/digests/digests.ts).
|
|
13
13
|
*
|
|
@@ -33,7 +33,7 @@ class DigestCache {
|
|
|
33
33
|
constructor(dbPath, opts = {}) {
|
|
34
34
|
if (!opts.readonly)
|
|
35
35
|
node_fs_1.default.mkdirSync(node_path_1.default.dirname(dbPath), { recursive: true });
|
|
36
|
-
this.db = new
|
|
36
|
+
this.db = new sqlite_1.default(dbPath, opts.readonly ? { readonly: true, fileMustExist: true } : undefined);
|
|
37
37
|
if (!opts.readonly) {
|
|
38
38
|
this.db.pragma('journal_mode = WAL');
|
|
39
39
|
this.db.exec(`
|
|
@@ -100,6 +100,10 @@ class DigestCache {
|
|
|
100
100
|
vectors.forEach((vector, i) => floats.set(vector, i * dims));
|
|
101
101
|
this.putVectorsStmt.run(key, space, dims, Buffer.from(floats.buffer));
|
|
102
102
|
}
|
|
103
|
+
/** The models the cache holds digests of, and how many digests each: what an eval run reads to arm the digests for the model the cache holds. */
|
|
104
|
+
digestsByModel() {
|
|
105
|
+
return this.db.prepare('SELECT model, COUNT(*) AS digests FROM note_digests GROUP BY model ORDER BY model').all();
|
|
106
|
+
}
|
|
103
107
|
/**
|
|
104
108
|
* Keep only the digests of the notes whose file hash is in `noteHashes`, and the vectors that go with them: a digest
|
|
105
109
|
* of text a note no longer holds (it was edited, forgotten, or its file is gone) is derived from content the user
|
package/dist/digests/digests.js
CHANGED
|
@@ -455,7 +455,7 @@ function rowTokens(item) {
|
|
|
455
455
|
const action = item.action?.trim() ? ` | ${item.action.trim()}` : '';
|
|
456
456
|
const amount = item.amount?.trim() ? ` | ${item.amount.trim()}` : '';
|
|
457
457
|
const date = item.date?.trim() ? ` | ${item.date.trim()}` : '';
|
|
458
|
-
return (0, o200k_1.
|
|
458
|
+
return (0, o200k_1.countO200k)(`- ${item.item}${action}${amount}${date}\n`);
|
|
459
459
|
}
|
|
460
460
|
/**
|
|
461
461
|
* The table of digest items for one search, and nothing else: the digests of the window's notes (first
|
|
@@ -530,7 +530,7 @@ async function digestTable(args) {
|
|
|
530
530
|
const ranked = candidates
|
|
531
531
|
.map(candidate => ({ candidate, score: cosine(queryVector, candidate.vector) }))
|
|
532
532
|
.sort((a, b) => b.score - a.score || a.candidate.order - b.candidate.order || a.candidate.index - b.candidate.index);
|
|
533
|
-
const frame = (0, o200k_1.
|
|
533
|
+
const frame = (0, o200k_1.countO200k)((0, attach_1.renderEvidenceTable)([]));
|
|
534
534
|
let used = frame;
|
|
535
535
|
const chosen = [];
|
|
536
536
|
for (const { candidate } of ranked) {
|
|
@@ -545,7 +545,7 @@ async function digestTable(args) {
|
|
|
545
545
|
report.status = items.length > 0 ? 'ok' : 'skipped';
|
|
546
546
|
report.itemsRendered = items.length;
|
|
547
547
|
report.text = items.length > 0 ? (0, attach_1.renderEvidenceTable)(items) : '';
|
|
548
|
-
report.tokens = items.length > 0 ? (0, o200k_1.
|
|
548
|
+
report.tokens = items.length > 0 ? (0, o200k_1.countO200k)(report.text) : 0;
|
|
549
549
|
report.items = chosen.map(c => ({ notePath: c.notePath, source: c.item.source, item: c.item.item }));
|
|
550
550
|
return { items, report };
|
|
551
551
|
}
|
package/dist/digests/worker.js
CHANGED
|
@@ -9,6 +9,8 @@ const node_fs_1 = __importDefault(require("node:fs"));
|
|
|
9
9
|
const sessionRouter_1 = require("../router/sessionRouter");
|
|
10
10
|
const digestCache_1 = require("./digestCache");
|
|
11
11
|
const digests_1 = require("./digests");
|
|
12
|
+
const log_1 = require("../mvp/log");
|
|
13
|
+
const log = (0, log_1.createLogger)('engine');
|
|
12
14
|
/**
|
|
13
15
|
* The production path of write-time digests (RENOR_DIGESTS, see digests.ts): a background worker that writes the
|
|
14
16
|
* digest of every note that has none, after a save, without ever being on the save's path.
|
|
@@ -116,7 +118,7 @@ class DigestWorker {
|
|
|
116
118
|
});
|
|
117
119
|
this.clearTimer = deps.clearTimer ?? (handle => clearTimeout(handle));
|
|
118
120
|
this.random = deps.random ?? Math.random;
|
|
119
|
-
this.log = deps.log ?? (line =>
|
|
121
|
+
this.log = deps.log ?? (line => log.info('digests_worker', { detail: line }));
|
|
120
122
|
}
|
|
121
123
|
/** Boot: digest what earlier saves, an import, or a workspace that slept left without a digest. */
|
|
122
124
|
start() {
|
|
@@ -11,7 +11,7 @@ exports.invalidateEmbeddingCache = invalidateEmbeddingCache;
|
|
|
11
11
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
12
12
|
const node_path_1 = __importDefault(require("node:path"));
|
|
13
13
|
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
14
|
-
const
|
|
14
|
+
const sqlite_1 = __importDefault(require("../db/sqlite"));
|
|
15
15
|
const IDENTITY_KIND = /^[\w.-]{1,32}$/;
|
|
16
16
|
const IDENTITY_MODEL = /^[\w./:-]{1,80}$/;
|
|
17
17
|
function resolveIdentity(inner, stated) {
|
|
@@ -69,7 +69,7 @@ class CachingEmbeddingProvider {
|
|
|
69
69
|
this.maxBytes = options.maxBytes ?? Infinity;
|
|
70
70
|
this.serving = options.serving === true;
|
|
71
71
|
node_fs_1.default.mkdirSync(node_path_1.default.dirname(options.cachePath), { recursive: true });
|
|
72
|
-
this.db = new
|
|
72
|
+
this.db = new sqlite_1.default(options.cachePath);
|
|
73
73
|
this.db.pragma('journal_mode = WAL');
|
|
74
74
|
// A serving read waits a moment for a lock, not ten seconds: past that the provider is the faster answer.
|
|
75
75
|
this.db.pragma(`busy_timeout = ${this.serving ? 100 : 10000}`);
|
|
@@ -283,7 +283,7 @@ function resolveEmbeddingCachePath(env = process.env) {
|
|
|
283
283
|
function describeEmbeddingCache(cachePath) {
|
|
284
284
|
if (!node_fs_1.default.existsSync(cachePath))
|
|
285
285
|
return [];
|
|
286
|
-
const db = new
|
|
286
|
+
const db = new sqlite_1.default(cachePath, { readonly: true });
|
|
287
287
|
try {
|
|
288
288
|
return db.prepare('SELECT namespace, COUNT(*) AS rows FROM embeddings GROUP BY namespace ORDER BY namespace').all();
|
|
289
289
|
}
|
|
@@ -300,7 +300,7 @@ function invalidateEmbeddingCache(cachePath, filter) {
|
|
|
300
300
|
throw new Error('invalidateEmbeddingCache: pass a namespace or all: true');
|
|
301
301
|
if (!node_fs_1.default.existsSync(cachePath))
|
|
302
302
|
return 0;
|
|
303
|
-
const db = new
|
|
303
|
+
const db = new sqlite_1.default(cachePath);
|
|
304
304
|
try {
|
|
305
305
|
db.pragma('busy_timeout = 10000');
|
|
306
306
|
return filter.namespace
|
|
@@ -317,7 +317,7 @@ class CacheOnlyVertex {
|
|
|
317
317
|
identity = { kind: 'vertex', model: 'gemini-embedding-001' };
|
|
318
318
|
stats = { hits: 0, misses: 0, upstreamTexts: 0 };
|
|
319
319
|
db;
|
|
320
|
-
constructor(file) { this.db = new
|
|
320
|
+
constructor(file) { this.db = new sqlite_1.default(file, { readonly: true, fileMustExist: true }); }
|
|
321
321
|
read(text, task) {
|
|
322
322
|
const namespace = 'vertex/gemini-embedding-001/768';
|
|
323
323
|
const key = node_crypto_1.default.createHash('sha256').update(`${namespace}\0${task}\0${this.dimensions}\0${text}`).digest('hex');
|
package/dist/embeddings/local.js
CHANGED
|
@@ -19,6 +19,7 @@ const MODEL_DIMENSIONS = {
|
|
|
19
19
|
'Xenova/multilingual-e5-large': 1024,
|
|
20
20
|
'intfloat/multilingual-e5-base': 768,
|
|
21
21
|
};
|
|
22
|
+
const TRANSFORMERS_PACKAGE = '@xenova/transformers';
|
|
22
23
|
class LocalEmbeddingProvider {
|
|
23
24
|
dimensions;
|
|
24
25
|
identity;
|
|
@@ -37,14 +38,15 @@ class LocalEmbeddingProvider {
|
|
|
37
38
|
return this.initPromise;
|
|
38
39
|
this.initPromise = (async () => {
|
|
39
40
|
// An optional peer dependency: an `npx renor-engine` install leaves it out (it is hundreds of megabytes), a clone has it.
|
|
40
|
-
const { pipeline, env } = await import(
|
|
41
|
+
const { pipeline, env } = await import(TRANSFORMERS_PACKAGE).catch((err) => {
|
|
41
42
|
if (!/cannot find (module|package)/i.test(String(err?.message)))
|
|
42
43
|
throw err;
|
|
43
44
|
throw new Error('EMBEDDING_PROVIDER=local needs the optional package @xenova/transformers: add it (`npx -y -p renor-engine -p @xenova/transformers renor-engine mcp`, or `npm install @xenova/transformers` beside a clone) or set EMBEDDING_PROVIDER=none');
|
|
44
45
|
});
|
|
45
46
|
env.cacheDir = path_1.default.join(os_1.default.homedir(), '.cache', 'osm-memory', 'models');
|
|
46
|
-
|
|
47
|
-
|
|
47
|
+
const extractor = await pipeline('feature-extraction', this.modelName);
|
|
48
|
+
this.extractor = extractor;
|
|
49
|
+
return extractor;
|
|
48
50
|
})();
|
|
49
51
|
return this.initPromise;
|
|
50
52
|
}
|
|
@@ -82,7 +82,8 @@ class OpenAIEmbeddingProvider {
|
|
|
82
82
|
return res.data.map(d => d.embedding);
|
|
83
83
|
}
|
|
84
84
|
catch (e) {
|
|
85
|
-
const
|
|
85
|
+
const err = e;
|
|
86
|
+
const status = err?.status;
|
|
86
87
|
if (status === 429 && attempt < 40) {
|
|
87
88
|
this.cooldown.set(ci, Date.now() + 60_000);
|
|
88
89
|
return this.embedWithRetry(batch, attempt + 1); // immediate retry on another key
|
|
@@ -93,8 +94,8 @@ class OpenAIEmbeddingProvider {
|
|
|
93
94
|
return this.embedWithRetry(batch, attempt + 1);
|
|
94
95
|
throw e;
|
|
95
96
|
}
|
|
96
|
-
if (status >= 500 && status < 600 && attempt < 10) {
|
|
97
|
-
const waitMs = Math.max(Number(
|
|
97
|
+
if (status !== undefined && status >= 500 && status < 600 && attempt < 10) {
|
|
98
|
+
const waitMs = Math.max(Number(err?.headers?.get?.('retry-after')) * 1000 || 0, 1000 * 2 ** attempt, 2000);
|
|
98
99
|
await new Promise(r => setTimeout(r, waitMs));
|
|
99
100
|
return this.embedWithRetry(batch, attempt + 1);
|
|
100
101
|
}
|
|
@@ -22,6 +22,8 @@
|
|
|
22
22
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
23
23
|
exports.SimdRows = exports.SIMD_MIN_BYTES = exports.SIMD_MAX_DIMS = void 0;
|
|
24
24
|
exports.simdScanEnabled = simdScanEnabled;
|
|
25
|
+
const log_1 = require("../mvp/log");
|
|
26
|
+
const log = (0, log_1.createLogger)('engine');
|
|
25
27
|
/** Rows one call writes distances for; the output area below holds this many f32. */
|
|
26
28
|
const CHUNK_ROWS = 8192;
|
|
27
29
|
/** The most floats in a row this module scans (the query area below). */
|
|
@@ -201,7 +203,7 @@ class SimdRows {
|
|
|
201
203
|
catch (err) {
|
|
202
204
|
if (!warned) {
|
|
203
205
|
warned = true;
|
|
204
|
-
|
|
206
|
+
log.warn('simd_scan_unavailable', { detail: 'scanning in JavaScript' }, err);
|
|
205
207
|
}
|
|
206
208
|
return null;
|
|
207
209
|
}
|