renor-engine 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +75 -0
- package/README.md +105 -2
- package/dist/.tsbuildinfo +1 -0
- package/dist/BUILD.json +6 -0
- package/dist/api/apiAuth.js +43 -0
- package/dist/api/server.js +174 -0
- package/dist/buildIdentity.js +149 -0
- package/dist/callPhases.js +82 -0
- package/dist/cli-extractors.js +46 -0
- package/dist/cli.js +500 -0
- package/dist/config.js +65 -0
- package/dist/context/composer.js +320 -0
- package/dist/context/readHonesty.js +478 -0
- package/dist/context/rowStore.js +238 -0
- package/dist/conversation/formatter.js +81 -0
- package/dist/conversation/parser.js +193 -0
- package/dist/conversation/types.js +2 -0
- package/dist/conversation/writer.js +42 -0
- package/dist/db/client.js +149 -0
- package/dist/db/handoffTable.js +31 -0
- package/dist/db/indexSalvage.js +101 -0
- package/dist/db/migrations.js +670 -0
- package/dist/db/readCache.js +109 -0
- package/dist/db/schema.js +5 -0
- package/dist/db/sqlite.js +396 -0
- package/dist/db/statements.js +89 -0
- package/dist/digests/digestCache.js +146 -0
- package/dist/digests/digests.js +597 -0
- package/dist/digests/worker.js +350 -0
- package/dist/embeddings/caching.js +337 -0
- package/dist/embeddings/coalescing.js +177 -0
- package/dist/embeddings/interactive.js +103 -0
- package/dist/embeddings/local.js +70 -0
- package/dist/embeddings/openai.js +106 -0
- package/dist/embeddings/rebuild.js +166 -0
- package/dist/embeddings/simdScan.js +260 -0
- package/dist/embeddings/space.js +21 -0
- package/dist/embeddings/types.js +14 -0
- package/dist/embeddings/vectorIndex.js +674 -0
- package/dist/embeddings/vertex.js +334 -0
- package/dist/eval/artifacts.js +42 -0
- package/dist/eval/cacheReport.js +146 -0
- package/dist/eval/comparison.js +182 -0
- package/dist/eval/context700-candidate.js +145 -0
- package/dist/eval/evalCache.js +568 -0
- package/dist/eval/hashEmbedding.js +35 -0
- package/dist/eval/indexSnapshots.js +548 -0
- package/dist/eval/longmemeval.js +56 -0
- package/dist/eval/metrics.js +181 -0
- package/dist/eval/pageDump.js +67 -0
- package/dist/eval/renorbenchBaseline.js +486 -0
- package/dist/eval/renorbenchDocuments.js +134 -0
- package/dist/eval/renorbenchProduct.js +182 -0
- package/dist/eval/runLedger.js +279 -0
- package/dist/eval/runManifest.js +485 -0
- package/dist/eval/scoring.js +197 -0
- package/dist/eval/textTokens.js +17 -0
- package/dist/evidence/attach.js +264 -0
- package/dist/evidence/background.js +147 -0
- package/dist/evidence/evidenceCache.js +182 -0
- package/dist/evidence/gate.js +37 -0
- package/dist/evidence/keyPool.js +85 -0
- package/dist/evidence/ollamaChat.js +84 -0
- package/dist/evidence/optin.js +165 -0
- package/dist/evidence/options.js +151 -0
- package/dist/evidence/secondPass.js +294 -0
- package/dist/extraction/classifyCapture.js +34 -0
- package/dist/extraction/decision-model.js +174 -0
- package/dist/extraction/eventExtractor.js +113 -0
- package/dist/extraction/extractionClient.js +122 -0
- package/dist/extraction/factExtractor.js +46 -0
- package/dist/extraction/factsWritePath.js +179 -0
- package/dist/extraction/preferenceExtractor.js +150 -0
- package/dist/extraction/secretGuard.js +176 -0
- package/dist/extraction/secretScanner.js +665 -0
- package/dist/extraction/typedFacts.js +224 -0
- package/dist/extraction/writePathAdjudication.js +110 -0
- package/dist/importing/apply.js +672 -0
- package/dist/importing/bundle-write.js +80 -0
- package/dist/importing/cli.js +408 -0
- package/dist/importing/existing.js +51 -0
- package/dist/importing/facts.js +121 -0
- package/dist/importing/forgetImported.js +79 -0
- package/dist/importing/hosted.js +467 -0
- package/dist/importing/journalLog.js +96 -0
- package/dist/importing/markdown.js +98 -0
- package/dist/importing/plan.js +201 -0
- package/dist/importing/secretScreen.js +113 -0
- package/dist/importing/serialize.js +188 -0
- package/dist/importing/source.js +463 -0
- package/dist/importing/types.js +6 -0
- package/dist/indexer/chunker.js +319 -0
- package/dist/indexer/dailyEntries.js +63 -0
- package/dist/indexer/hasher.js +10 -0
- package/dist/indexer/laneRuns.js +48 -0
- package/dist/indexer/o200k.js +481 -0
- package/dist/indexer/o200k.json +8 -0
- package/dist/indexer/outsideRoot.js +77 -0
- package/dist/indexer/parser.js +170 -0
- package/dist/indexer/pipeline.js +988 -0
- package/dist/indexer/staleness.js +201 -0
- package/dist/indexer/tokenizer.js +77 -0
- package/dist/indexer/walker.js +170 -0
- package/dist/mcp/hive.js +88 -0
- package/dist/mcp/requestErrors.js +47 -0
- package/dist/mcp/server.js +426 -0
- package/dist/memory/agentLease.js +43 -0
- package/dist/memory/confinedWrite.js +315 -0
- package/dist/memory/controls.js +73 -0
- package/dist/memory/dailyNote.js +265 -0
- package/dist/memory/durableAppend.js +57 -0
- package/dist/memory/factConflicts.js +203 -0
- package/dist/memory/factTerms.js +321 -0
- package/dist/memory/handoff.js +363 -0
- package/dist/memory/hiveNameHeader.cjs +10 -0
- package/dist/memory/inventory.js +55 -0
- package/dist/memory/invisibleText.js +20 -0
- package/dist/memory/lifecycle.js +1991 -0
- package/dist/memory/limits.js +13 -0
- package/dist/memory/requests.js +183 -0
- package/dist/memory/sourceName.js +276 -0
- package/dist/memory/statusFacts.js +61 -0
- package/dist/memory/whatsNew.js +158 -0
- package/dist/memory/writer.js +175 -0
- package/dist/memory/writerConflicts.js +85 -0
- package/dist/memorymap/memoryMap.js +1594 -0
- package/dist/memorymap/replacedJournal.js +58 -0
- package/dist/memorymap/replacements.js +410 -0
- package/dist/mvp/activity.js +86 -0
- package/dist/mvp/agentMemory.js +1800 -0
- package/dist/mvp/answerCheck.js +211 -0
- package/dist/mvp/auth.js +531 -0
- package/dist/mvp/cli.js +75 -0
- package/dist/mvp/client-store.js +462 -0
- package/dist/mvp/compileCache.js +242 -0
- package/dist/mvp/compileSeed.js +33 -0
- package/dist/mvp/config.js +288 -0
- package/dist/mvp/connectionStatus.js +80 -0
- package/dist/mvp/contextDistinct.js +53 -0
- package/dist/mvp/contextProfile.js +87 -0
- package/dist/mvp/contextRelevance.js +413 -0
- package/dist/mvp/directorySurface.js +68 -0
- package/dist/mvp/engine.js +1078 -0
- package/dist/mvp/exportAll.js +374 -0
- package/dist/mvp/factLanePack.js +77 -0
- package/dist/mvp/factMarkers.js +65 -0
- package/dist/mvp/factsLane.js +203 -0
- package/dist/mvp/hiddenSpans.js +585 -0
- package/dist/mvp/history.js +414 -0
- package/dist/mvp/hitDates.js +31 -0
- package/dist/mvp/hitFloor.js +36 -0
- package/dist/mvp/identity.js +18 -0
- package/dist/mvp/idle.js +113 -0
- package/dist/mvp/log.js +118 -0
- package/dist/mvp/memoryDir.js +98 -0
- package/dist/mvp/posthogMcp.js +210 -0
- package/dist/mvp/readScope.js +104 -0
- package/dist/mvp/readUsefulness.js +222 -0
- package/dist/mvp/sanitize.js +48 -0
- package/dist/mvp/schemaRejections.js +91 -0
- package/dist/mvp/scope.js +127 -0
- package/dist/mvp/server.js +2604 -0
- package/dist/mvp/serverCard.js +38 -0
- package/dist/mvp/storedSize.js +139 -0
- package/dist/mvp/telemetry.js +474 -0
- package/dist/net/poolFetch.js +174 -0
- package/dist/recall/actionRecall.js +282 -0
- package/dist/recall/autoRecall.js +447 -0
- package/dist/recall/searchRecall.js +189 -0
- package/dist/retrieval/adviceLines.js +195 -0
- package/dist/retrieval/adviceQuestion.js +44 -0
- package/dist/retrieval/conversationShape.js +44 -0
- package/dist/retrieval/dailyEntryTime.js +55 -0
- package/dist/retrieval/entityLookup.js +162 -0
- package/dist/retrieval/factsFirst.js +188 -0
- package/dist/retrieval/keyword.js +84 -0
- package/dist/retrieval/ledgerEventTime.js +51 -0
- package/dist/retrieval/ledgerLiveness.js +83 -0
- package/dist/retrieval/ledgerPresentation.js +161 -0
- package/dist/retrieval/noteIdentity.js +57 -0
- package/dist/retrieval/orchestrator.js +1171 -0
- package/dist/retrieval/plainWording.js +236 -0
- package/dist/retrieval/queryHelpers.js +142 -0
- package/dist/retrieval/questionSpan.js +475 -0
- package/dist/retrieval/readerEvents.js +690 -0
- package/dist/retrieval/readerLayout.js +422 -0
- package/dist/retrieval/relativeTimeHints.js +113 -0
- package/dist/retrieval/semantic.js +67 -0
- package/dist/router/routerCache.js +106 -0
- package/dist/router/sessionRouter.js +726 -0
- package/dist/toolRefusal.js +120 -0
- package/dist/watcher/watcher.js +77 -0
- package/package.json +118 -4
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.tableHasColumns = tableHasColumns;
|
|
4
|
+
exports.inReadSnapshot = inReadSnapshot;
|
|
5
|
+
exports.databaseStamp = databaseStamp;
|
|
6
|
+
exports.readUntilWrite = readUntilWrite;
|
|
7
|
+
const statements_1 = require("./statements");
|
|
8
|
+
/**
|
|
9
|
+
* Two things a read asks of the database again and again and that change rarely: the shape of a table, and a value
|
|
10
|
+
* derived from every row of one (the ledger's closed events). A `get_context` asked for each of them six times.
|
|
11
|
+
*/
|
|
12
|
+
const withColumns = new WeakMap();
|
|
13
|
+
/**
|
|
14
|
+
* Whether `table` has every one of `columns` (the reads that predate a column ask, so an old index still answers).
|
|
15
|
+
* Only a yes is kept for the connection: a table that lacks a column may get it from a migration, and a column that
|
|
16
|
+
* is there is never dropped, so a no is asked again each time and a yes never is. The names are the code's own, never
|
|
17
|
+
* a caller's.
|
|
18
|
+
*/
|
|
19
|
+
function tableHasColumns(db, table, columns) {
|
|
20
|
+
const key = `${table}(${columns.join(',')})`;
|
|
21
|
+
let known = withColumns.get(db);
|
|
22
|
+
if (known?.has(key))
|
|
23
|
+
return true;
|
|
24
|
+
const present = new Set(db.prepare(`PRAGMA table_info(${table})`).all().map(c => c.name));
|
|
25
|
+
if (!columns.every(c => present.has(c)))
|
|
26
|
+
return false;
|
|
27
|
+
if (!known)
|
|
28
|
+
withColumns.set(db, known = new Map());
|
|
29
|
+
known.set(key, true);
|
|
30
|
+
return true;
|
|
31
|
+
}
|
|
32
|
+
const kept = new WeakMap();
|
|
33
|
+
/** Values kept per connection. A key may carry what a caller sent (a scope), so the oldest goes when the room is used. */
|
|
34
|
+
const MAX_KEPT = 64;
|
|
35
|
+
/** The connections inside a snapshot (`inReadSnapshot`). */
|
|
36
|
+
const snapshots = new WeakSet();
|
|
37
|
+
/**
|
|
38
|
+
* What the database has been through: changes this connection committed (`total_changes()`, which counts every row an
|
|
39
|
+
* INSERT, UPDATE or DELETE touched, triggers and virtual tables included) and changes any other connection committed
|
|
40
|
+
* (`PRAGMA data_version`, which a connection's own writes do not move). Both only ever go up, so two equal stamps are
|
|
41
|
+
* the same data. Null inside a transaction: it can still be rolled back, and its changes would be counted for good. A
|
|
42
|
+
* snapshot is the exception: it is ended with a COMMIT and never rolled back, so what it has counted stays counted.
|
|
43
|
+
*/
|
|
44
|
+
function stampOf(db) {
|
|
45
|
+
if (db.inTransaction && !snapshots.has(db))
|
|
46
|
+
return null;
|
|
47
|
+
const version = (0, statements_1.prepareOnce)(db, 'PRAGMA data_version').pluck().get();
|
|
48
|
+
const changes = (0, statements_1.prepareOnce)(db, 'SELECT total_changes()').pluck().get();
|
|
49
|
+
return `${version}:${changes}`;
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Runs `read` in ONE transaction. A statement that runs on its own begins a read transaction and ends it again, and in WAL
|
|
53
|
+
* mode that is two locks on the shared-memory index and a file-size call: about 5 microseconds for a lookup that takes 1.5,
|
|
54
|
+
* and a `get_context` runs a hundred of them. Inside a transaction the lock is taken once, and every statement after the first
|
|
55
|
+
* pays for what it reads. The reads see the database as it was when the first of them began; a commit by another process
|
|
56
|
+
* meanwhile is in the next read, not in a mix of the two.
|
|
57
|
+
*
|
|
58
|
+
* `read` is for a stretch of reads that holds no `await` and writes nothing: nothing else of this process can run in a
|
|
59
|
+
* stretch that does not yield, and a write inside it would not be durable until the transaction ended. When a transaction is
|
|
60
|
+
* open already (a write transaction, or a snapshot outside this one) `read` just runs in it. The stamps (`databaseStamp`,
|
|
61
|
+
* `readUntilWrite`) hold for a snapshot, which is committed and never rolled back.
|
|
62
|
+
*/
|
|
63
|
+
function inReadSnapshot(db, read) {
|
|
64
|
+
if (db.inTransaction)
|
|
65
|
+
return read();
|
|
66
|
+
(0, statements_1.prepareOnce)(db, 'BEGIN').run();
|
|
67
|
+
snapshots.add(db);
|
|
68
|
+
let done = false;
|
|
69
|
+
try {
|
|
70
|
+
const result = read();
|
|
71
|
+
done = true;
|
|
72
|
+
return result;
|
|
73
|
+
}
|
|
74
|
+
finally {
|
|
75
|
+
snapshots.delete(db);
|
|
76
|
+
try {
|
|
77
|
+
(0, statements_1.prepareOnce)(db, 'COMMIT').run();
|
|
78
|
+
}
|
|
79
|
+
catch (err) {
|
|
80
|
+
if (done)
|
|
81
|
+
throw err; // the read failed first: that is the error to report
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
/** The stamp `readUntilWrite` keeps a value under. Null inside a transaction, which can still roll back. */
|
|
86
|
+
function databaseStamp(db) {
|
|
87
|
+
return stampOf(db);
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* `compute()` for the database as it is now, read once and kept until anything is written to it, by this connection or
|
|
91
|
+
* another. The same answer a fresh `compute()` gives: nothing is kept across a write, and nothing at all inside a
|
|
92
|
+
* transaction. `compute` must read only the database, and what it returns is shared: callers must not change it.
|
|
93
|
+
*/
|
|
94
|
+
function readUntilWrite(db, key, compute) {
|
|
95
|
+
const stamp = stampOf(db);
|
|
96
|
+
if (stamp === null)
|
|
97
|
+
return compute();
|
|
98
|
+
let byKey = kept.get(db);
|
|
99
|
+
if (!byKey)
|
|
100
|
+
kept.set(db, byKey = new Map());
|
|
101
|
+
const held = byKey.get(key);
|
|
102
|
+
if (held && held.stamp === stamp)
|
|
103
|
+
return held.value;
|
|
104
|
+
const value = compute();
|
|
105
|
+
if (!byKey.has(key) && byKey.size >= MAX_KEPT)
|
|
106
|
+
byKey.delete(byKey.keys().next().value);
|
|
107
|
+
byKey.set(key, { stamp, value });
|
|
108
|
+
return value;
|
|
109
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.runMigrations = void 0;
|
|
4
|
+
var migrations_1 = require("./migrations");
|
|
5
|
+
Object.defineProperty(exports, "runMigrations", { enumerable: true, get: function () { return migrations_1.runMigrations; } });
|
|
@@ -0,0 +1,396 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.Statement = exports.MIN_NODE = void 0;
|
|
7
|
+
exports.nodeSupportProblem = nodeSupportProblem;
|
|
8
|
+
exports.sqliteRuntimeProblem = sqliteRuntimeProblem;
|
|
9
|
+
/**
|
|
10
|
+
* The engine's SQLite driver: Node's built-in `node:sqlite`, behind the part of the better-sqlite3 API the engine uses.
|
|
11
|
+
*
|
|
12
|
+
* Why not better-sqlite3: it is a native addon that builds itself in an npm install script. `npx -y renor-engine` then
|
|
13
|
+
* breaks for anyone whose npm does not run dependency scripts (`ignore-scripts=true`, newer npm defaults), and on a Node
|
|
14
|
+
* with no prebuilt binary the install falls back to compiling C++ for over a minute while the MCP client gives up. Node
|
|
15
|
+
* ships SQLite itself (FTS5 included) and loads sqlite-vec, whose prebuilt binary is a plain optional dependency, so the
|
|
16
|
+
* engine installs with no script at all.
|
|
17
|
+
*
|
|
18
|
+
* Behaviour kept from better-sqlite3, because the engine and its tests rely on it:
|
|
19
|
+
* - rows are plain objects and BLOBs come back as Buffer; `pluck()`, `raw()` and `safeIntegers()` work as before;
|
|
20
|
+
* - `transaction(fn)` begins (or, nested, takes a savepoint), commits, and rolls back on a throw;
|
|
21
|
+
* - errors carry the SQLite code name (`SQLITE_CONSTRAINT_UNIQUE`, `SQLITE_CORRUPT`, ...) in `code`;
|
|
22
|
+
* - a 5 s busy timeout, a 16 MB page cache, and `synchronous = NORMAL` on a connection that opens a database already in
|
|
23
|
+
* WAL mode (better-sqlite3's compiled defaults; see matchWalSynchronous);
|
|
24
|
+
* Extension loading is allowed through the C API only (`loadExtension`, for sqlite-vec); SQL's `load_extension()` stays
|
|
25
|
+
* refused, as it was.
|
|
26
|
+
*/
|
|
27
|
+
const fs_1 = __importDefault(require("fs"));
|
|
28
|
+
/** The first Node with every `node:sqlite` feature used here (extension loading, `isTransaction`, `setReturnArrays`, `backup`). */
|
|
29
|
+
exports.MIN_NODE = '22.16.0';
|
|
30
|
+
/** Why this Node cannot run the engine, or null when it can. */
|
|
31
|
+
function nodeSupportProblem(version = process.versions.node) {
|
|
32
|
+
const [major, minor] = version.split('.').map(Number);
|
|
33
|
+
if (major >= 24 || (major === 22 && minor >= 16))
|
|
34
|
+
return null;
|
|
35
|
+
return `Renor needs Node ${exports.MIN_NODE} or newer (it uses Node's built-in SQLite); this is Node ${version}.`;
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Why the engine cannot store anything here, as one line with its fix, or null when it can: this Node has no usable
|
|
39
|
+
* built-in SQLite, or sqlite-vec's prebuilt binary for this platform is missing or does not load. The CLI says this and
|
|
40
|
+
* exits before it starts, so an MCP client shows the reason instead of a connection that silently failed.
|
|
41
|
+
*/
|
|
42
|
+
function sqliteRuntimeProblem() {
|
|
43
|
+
const node = nodeSupportProblem();
|
|
44
|
+
if (node)
|
|
45
|
+
return `${node} Fix: install Node 22 LTS or newer (https://nodejs.org), then run the command again.`;
|
|
46
|
+
try {
|
|
47
|
+
const db = new SqliteDatabase(':memory:');
|
|
48
|
+
try {
|
|
49
|
+
require('sqlite-vec').load(db);
|
|
50
|
+
}
|
|
51
|
+
finally {
|
|
52
|
+
db.close();
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
const reason = (err instanceof Error ? err.message.split('\n')[0] : String(err)).replace(/\.?\s*$/, '.');
|
|
57
|
+
return `the vector search extension (sqlite-vec) did not load on ${process.platform}-${process.arch}: ${reason} `
|
|
58
|
+
+ 'Fix: let npm install optional dependencies (remove omit=optional or --no-optional from your npm settings), '
|
|
59
|
+
+ 'then run `npx -y renor-engine@latest mcp` again.';
|
|
60
|
+
}
|
|
61
|
+
return null;
|
|
62
|
+
}
|
|
63
|
+
let sqlite;
|
|
64
|
+
function nodeSqlite() {
|
|
65
|
+
if (sqlite)
|
|
66
|
+
return sqlite;
|
|
67
|
+
const problem = nodeSupportProblem();
|
|
68
|
+
if (problem)
|
|
69
|
+
throw new Error(`${problem} Install Node 22 LTS or newer from https://nodejs.org and run the command again.`);
|
|
70
|
+
// Node 22 prints "SQLite is an experimental feature" once on load; to an MCP client that is noise on stderr, not news.
|
|
71
|
+
const emitWarning = process.emitWarning;
|
|
72
|
+
process.emitWarning = ((warning, ...rest) => {
|
|
73
|
+
if (String(warning instanceof Error ? warning.message : warning).includes('SQLite is an experimental feature'))
|
|
74
|
+
return;
|
|
75
|
+
return emitWarning.call(process, warning, ...rest);
|
|
76
|
+
});
|
|
77
|
+
try {
|
|
78
|
+
sqlite = require('node:sqlite');
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
process.emitWarning = emitWarning;
|
|
82
|
+
}
|
|
83
|
+
return sqlite;
|
|
84
|
+
}
|
|
85
|
+
/** SQLite's primary result codes, by number. */
|
|
86
|
+
const PRIMARY_CODES = ('OK ERROR INTERNAL PERM ABORT BUSY LOCKED NOMEM READONLY INTERRUPT IOERR CORRUPT NOTFOUND FULL CANTOPEN PROTOCOL EMPTY '
|
|
87
|
+
+ 'SCHEMA TOOBIG CONSTRAINT MISMATCH MISUSE NOLFS AUTH FORMAT RANGE NOTADB NOTICE WARNING').split(' ');
|
|
88
|
+
/** The extended result codes worth telling apart; any other extended code is named by its primary code. */
|
|
89
|
+
const EXTENDED_CODES = new Map(('261 BUSY_RECOVERY,517 BUSY_SNAPSHOT,773 BUSY_TIMEOUT,262 LOCKED_SHAREDCACHE,267 CORRUPT_VTAB,523 CORRUPT_SEQUENCE,'
|
|
90
|
+
+ '779 CORRUPT_INDEX,264 READONLY_RECOVERY,520 READONLY_CANTLOCK,776 READONLY_ROLLBACK,1032 READONLY_DBMOVED,'
|
|
91
|
+
+ '275 CONSTRAINT_CHECK,531 CONSTRAINT_COMMITHOOK,787 CONSTRAINT_FOREIGNKEY,1043 CONSTRAINT_FUNCTION,'
|
|
92
|
+
+ '1299 CONSTRAINT_NOTNULL,1555 CONSTRAINT_PRIMARYKEY,1811 CONSTRAINT_TRIGGER,2067 CONSTRAINT_UNIQUE,'
|
|
93
|
+
+ '2323 CONSTRAINT_VTAB,2579 CONSTRAINT_ROWID,3091 CONSTRAINT_DATATYPE').split(',').map(pair => [Number(pair.split(' ')[0]), pair.split(' ')[1]]));
|
|
94
|
+
/** Gives a `node:sqlite` error better-sqlite3's `code` (`SQLITE_<NAME>`); every other error passes through unchanged. */
|
|
95
|
+
function withSqliteCode(err) {
|
|
96
|
+
const e = err;
|
|
97
|
+
if (e && e.code === 'ERR_SQLITE_ERROR' && typeof e.errcode === 'number') {
|
|
98
|
+
const name = EXTENDED_CODES.get(e.errcode) ?? PRIMARY_CODES[e.errcode & 0xff] ?? 'ERROR';
|
|
99
|
+
try {
|
|
100
|
+
e.code = `SQLITE_${name}`;
|
|
101
|
+
}
|
|
102
|
+
catch { /* a frozen error keeps its code */ }
|
|
103
|
+
}
|
|
104
|
+
return err;
|
|
105
|
+
}
|
|
106
|
+
function guarded(work) {
|
|
107
|
+
try {
|
|
108
|
+
return work();
|
|
109
|
+
}
|
|
110
|
+
catch (err) {
|
|
111
|
+
throw withSqliteCode(err);
|
|
112
|
+
}
|
|
113
|
+
}
|
|
114
|
+
/** Every bigint in a row (or rows) as the nearest double, as better-sqlite3 read an integer beyond 2^53. */
|
|
115
|
+
function roundBigInts(value) {
|
|
116
|
+
if (typeof value === 'bigint')
|
|
117
|
+
return Number(value);
|
|
118
|
+
if (Array.isArray(value))
|
|
119
|
+
return value.map(roundBigInts);
|
|
120
|
+
if (value !== null && typeof value === 'object' && !ArrayBuffer.isView(value)) {
|
|
121
|
+
const out = {};
|
|
122
|
+
for (const key in value)
|
|
123
|
+
out[key] = roundBigInts(value[key]);
|
|
124
|
+
return out;
|
|
125
|
+
}
|
|
126
|
+
return value;
|
|
127
|
+
}
|
|
128
|
+
function asBuffer(value) {
|
|
129
|
+
return value instanceof Uint8Array && !Buffer.isBuffer(value) ? Buffer.from(value.buffer, value.byteOffset, value.byteLength) : value;
|
|
130
|
+
}
|
|
131
|
+
/** better-sqlite3 takes named parameters as an object in any position and positional ones also as arrays. */
|
|
132
|
+
function bindArgs(params) {
|
|
133
|
+
let named;
|
|
134
|
+
const positional = [];
|
|
135
|
+
for (const p of params) {
|
|
136
|
+
if (Array.isArray(p))
|
|
137
|
+
positional.push(...p);
|
|
138
|
+
else if (p !== null && typeof p === 'object' && !ArrayBuffer.isView(p))
|
|
139
|
+
named = p;
|
|
140
|
+
else
|
|
141
|
+
positional.push(p);
|
|
142
|
+
}
|
|
143
|
+
return named === undefined ? positional : [named, ...positional];
|
|
144
|
+
}
|
|
145
|
+
/** SQL that can change a table's columns, after which this connection's prepared statements are prepared again. */
|
|
146
|
+
const SCHEMA_CHANGE = /\b(ALTER|CREATE|DROP)\b/i;
|
|
147
|
+
const WRITE_OPCODES = new Set(['JournalMode', 'Vacuum', 'VCreate', 'VDestroy', 'VUpdate', 'ParseSchema']);
|
|
148
|
+
class Statement {
|
|
149
|
+
database;
|
|
150
|
+
stmt;
|
|
151
|
+
source;
|
|
152
|
+
mode = 'object';
|
|
153
|
+
bigInts = false;
|
|
154
|
+
changesSchema;
|
|
155
|
+
generation;
|
|
156
|
+
constructor(database, stmt, source) {
|
|
157
|
+
this.database = database;
|
|
158
|
+
this.stmt = stmt;
|
|
159
|
+
this.source = source;
|
|
160
|
+
this.changesSchema = SCHEMA_CHANGE.test(source);
|
|
161
|
+
this.generation = database.schemaGeneration;
|
|
162
|
+
}
|
|
163
|
+
run(...params) {
|
|
164
|
+
const result = guarded(() => this.current().run(...bindArgs(params)));
|
|
165
|
+
if (this.changesSchema)
|
|
166
|
+
this.database.schemaChanged();
|
|
167
|
+
// `changes` is a number even under safeIntegers(), as in better-sqlite3; only lastInsertRowid becomes a bigint.
|
|
168
|
+
return { changes: Number(result.changes), lastInsertRowid: result.lastInsertRowid };
|
|
169
|
+
}
|
|
170
|
+
get(...params) {
|
|
171
|
+
const row = this.lossy(() => this.current().get(...bindArgs(params)));
|
|
172
|
+
return row === undefined ? undefined : this.shape(row);
|
|
173
|
+
}
|
|
174
|
+
all(...params) {
|
|
175
|
+
return this.lossy(() => this.current().all(...bindArgs(params))).map(row => this.shape(row));
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Errors raised while stepping get the same `SQLITE_*` code as the other calls. An integer beyond 2^53 still throws
|
|
179
|
+
* RangeError here (better-sqlite3 returned it rounded); use safeIntegers(), or get()/all(), which round as before.
|
|
180
|
+
*/
|
|
181
|
+
*iterate(...params) {
|
|
182
|
+
const rows = guarded(() => this.current().iterate(...bindArgs(params)));
|
|
183
|
+
try {
|
|
184
|
+
for (;;) {
|
|
185
|
+
const step = guarded(() => rows.next());
|
|
186
|
+
if (step.done)
|
|
187
|
+
return;
|
|
188
|
+
yield this.shape(step.value);
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
finally {
|
|
192
|
+
rows.return?.();
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
pluck(toggleState = true) {
|
|
196
|
+
return this.setMode(toggleState ? 'pluck' : this.mode === 'pluck' ? 'object' : this.mode);
|
|
197
|
+
}
|
|
198
|
+
raw(toggleState = true) {
|
|
199
|
+
return this.setMode(toggleState ? 'raw' : this.mode === 'raw' ? 'object' : this.mode);
|
|
200
|
+
}
|
|
201
|
+
safeIntegers(toggleState = true) {
|
|
202
|
+
this.bigInts = toggleState;
|
|
203
|
+
this.stmt.setReadBigInts(toggleState);
|
|
204
|
+
return this;
|
|
205
|
+
}
|
|
206
|
+
/**
|
|
207
|
+
* Whether the statement writes nothing, as SQLite's sqlite3_stmt_readonly() decides it: its program opens no write
|
|
208
|
+
* transaction and changes no journal mode, virtual table or file. Read from the statement's EXPLAIN on demand.
|
|
209
|
+
*/
|
|
210
|
+
get readonly() {
|
|
211
|
+
const program = guarded(() => this.database.prepareNative(`EXPLAIN ${this.source}`).all());
|
|
212
|
+
return !program.some(op => (op.opcode === 'Transaction' && op.p2 > 0) || WRITE_OPCODES.has(op.opcode));
|
|
213
|
+
}
|
|
214
|
+
columns() {
|
|
215
|
+
return this.current().columns();
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* better-sqlite3 returned an integer beyond 2^53 rounded to the nearest double; node:sqlite throws RangeError
|
|
219
|
+
* (ERR_OUT_OF_RANGE) instead. A statement that writes nothing is read again as bigints and rounded the same way; a
|
|
220
|
+
* statement that writes is never run twice, so its RangeError stands.
|
|
221
|
+
*/
|
|
222
|
+
lossy(read) {
|
|
223
|
+
try {
|
|
224
|
+
return guarded(read);
|
|
225
|
+
}
|
|
226
|
+
catch (err) {
|
|
227
|
+
if (this.bigInts || err.code !== 'ERR_OUT_OF_RANGE' || !this.readonly)
|
|
228
|
+
throw err;
|
|
229
|
+
this.stmt.setReadBigInts(true);
|
|
230
|
+
try {
|
|
231
|
+
return roundBigInts(guarded(read));
|
|
232
|
+
}
|
|
233
|
+
finally {
|
|
234
|
+
this.stmt.setReadBigInts(false);
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
/**
|
|
239
|
+
* The native statement, prepared again after this connection changed the schema. SQLite re-prepares a statement by
|
|
240
|
+
* itself, but `node:sqlite` reads the result's columns before that happens, so the first run after an ALTER TABLE
|
|
241
|
+
* would return the old columns; better-sqlite3 did not.
|
|
242
|
+
*/
|
|
243
|
+
current() {
|
|
244
|
+
if (this.generation !== this.database.schemaGeneration) {
|
|
245
|
+
this.stmt = this.database.prepareNative(this.source);
|
|
246
|
+
this.stmt.setReturnArrays(this.mode !== 'object');
|
|
247
|
+
this.stmt.setReadBigInts(this.bigInts);
|
|
248
|
+
this.generation = this.database.schemaGeneration;
|
|
249
|
+
}
|
|
250
|
+
return this.stmt;
|
|
251
|
+
}
|
|
252
|
+
setMode(mode) {
|
|
253
|
+
this.mode = mode;
|
|
254
|
+
this.stmt.setReturnArrays(mode !== 'object');
|
|
255
|
+
return this;
|
|
256
|
+
}
|
|
257
|
+
shape(row) {
|
|
258
|
+
if (this.mode === 'pluck')
|
|
259
|
+
return asBuffer(row[0]);
|
|
260
|
+
if (this.mode === 'raw')
|
|
261
|
+
return row.map(asBuffer);
|
|
262
|
+
const out = {};
|
|
263
|
+
for (const key in row)
|
|
264
|
+
out[key] = asBuffer(row[key]);
|
|
265
|
+
return out;
|
|
266
|
+
}
|
|
267
|
+
}
|
|
268
|
+
exports.Statement = Statement;
|
|
269
|
+
class SqliteDatabase {
|
|
270
|
+
name;
|
|
271
|
+
readonly;
|
|
272
|
+
memory;
|
|
273
|
+
db;
|
|
274
|
+
closed = false;
|
|
275
|
+
extensions = new Set();
|
|
276
|
+
/** @internal Bumped whenever this connection may have changed the schema; see Statement's `current()`. */
|
|
277
|
+
schemaGeneration = 0;
|
|
278
|
+
constructor(filename = '', options = {}) {
|
|
279
|
+
this.name = filename;
|
|
280
|
+
this.readonly = options.readonly === true;
|
|
281
|
+
this.memory = filename === ':memory:' || filename === '';
|
|
282
|
+
if (options.fileMustExist && !this.memory && !fs_1.default.existsSync(filename)) {
|
|
283
|
+
throw Object.assign(new Error(`unable to open database file: ${filename}`), { code: 'SQLITE_CANTOPEN' });
|
|
284
|
+
}
|
|
285
|
+
const { DatabaseSync } = nodeSqlite();
|
|
286
|
+
this.db = guarded(() => new DatabaseSync(filename, { readOnly: this.readonly, allowExtension: true }));
|
|
287
|
+
try {
|
|
288
|
+
this.db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.trunc(options.timeout ?? 5000))}`);
|
|
289
|
+
this.db.exec('PRAGMA cache_size = -16000');
|
|
290
|
+
this.matchWalSynchronous();
|
|
291
|
+
}
|
|
292
|
+
catch (err) {
|
|
293
|
+
this.db.close();
|
|
294
|
+
throw withSqliteCode(err);
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
get open() {
|
|
298
|
+
return !this.closed;
|
|
299
|
+
}
|
|
300
|
+
get inTransaction() {
|
|
301
|
+
return !this.closed && this.db.isTransaction;
|
|
302
|
+
}
|
|
303
|
+
prepare(source) {
|
|
304
|
+
return new Statement(this, guarded(() => this.db.prepare(source)), source);
|
|
305
|
+
}
|
|
306
|
+
/** @internal The `node:sqlite` statement itself, for Statement's own bookkeeping. */
|
|
307
|
+
prepareNative(source) {
|
|
308
|
+
return this.db.prepare(source);
|
|
309
|
+
}
|
|
310
|
+
exec(source) {
|
|
311
|
+
guarded(() => this.db.exec(source));
|
|
312
|
+
if (SCHEMA_CHANGE.test(source))
|
|
313
|
+
this.schemaChanged();
|
|
314
|
+
return this;
|
|
315
|
+
}
|
|
316
|
+
/** @internal */
|
|
317
|
+
schemaChanged() {
|
|
318
|
+
this.schemaGeneration++;
|
|
319
|
+
}
|
|
320
|
+
pragma(source, options = {}) {
|
|
321
|
+
const stmt = guarded(() => this.db.prepare(`PRAGMA ${source}`));
|
|
322
|
+
if (options.simple) {
|
|
323
|
+
stmt.setReturnArrays(true);
|
|
324
|
+
const row = guarded(() => stmt.get());
|
|
325
|
+
return row === undefined ? undefined : asBuffer(row[0]);
|
|
326
|
+
}
|
|
327
|
+
return guarded(() => stmt.all()).map(row => ({ ...row }));
|
|
328
|
+
}
|
|
329
|
+
transaction(fn) {
|
|
330
|
+
const wrap = (begin) => {
|
|
331
|
+
const db = this;
|
|
332
|
+
return function sqliteTransaction(...args) {
|
|
333
|
+
const nested = db.inTransaction;
|
|
334
|
+
db.exec(nested ? 'SAVEPOINT "_renor_tx"' : begin);
|
|
335
|
+
try {
|
|
336
|
+
const result = fn.apply(this, args);
|
|
337
|
+
if (result && typeof result.then === 'function') {
|
|
338
|
+
throw new TypeError('Transaction function cannot return a promise');
|
|
339
|
+
}
|
|
340
|
+
db.exec(nested ? 'RELEASE "_renor_tx"' : 'COMMIT');
|
|
341
|
+
return result;
|
|
342
|
+
}
|
|
343
|
+
catch (err) {
|
|
344
|
+
if (db.inTransaction) {
|
|
345
|
+
if (nested)
|
|
346
|
+
db.exec('ROLLBACK TO "_renor_tx"; RELEASE "_renor_tx"');
|
|
347
|
+
else
|
|
348
|
+
db.exec('ROLLBACK');
|
|
349
|
+
}
|
|
350
|
+
throw err;
|
|
351
|
+
}
|
|
352
|
+
};
|
|
353
|
+
};
|
|
354
|
+
const tx = wrap('BEGIN');
|
|
355
|
+
tx.default = tx;
|
|
356
|
+
tx.deferred = wrap('BEGIN DEFERRED');
|
|
357
|
+
tx.immediate = wrap('BEGIN IMMEDIATE');
|
|
358
|
+
tx.exclusive = wrap('BEGIN EXCLUSIVE');
|
|
359
|
+
return tx;
|
|
360
|
+
}
|
|
361
|
+
/**
|
|
362
|
+
* Loads an extension once per connection: a second load only registers the same functions again, and that fails
|
|
363
|
+
* ("error during initialization") inside a transaction that has deleted rows behind a virtual table, where
|
|
364
|
+
* src/indexer/outsideRoot.ts loads sqlite-vec.
|
|
365
|
+
*/
|
|
366
|
+
loadExtension(path) {
|
|
367
|
+
if (this.extensions.has(path))
|
|
368
|
+
return this;
|
|
369
|
+
guarded(() => this.db.loadExtension(path));
|
|
370
|
+
this.extensions.add(path);
|
|
371
|
+
return this;
|
|
372
|
+
}
|
|
373
|
+
backup(destinationFile) {
|
|
374
|
+
return nodeSqlite().backup(this.db, destinationFile);
|
|
375
|
+
}
|
|
376
|
+
close() {
|
|
377
|
+
if (!this.closed) {
|
|
378
|
+
this.closed = true;
|
|
379
|
+
this.db.close();
|
|
380
|
+
}
|
|
381
|
+
return this;
|
|
382
|
+
}
|
|
383
|
+
/**
|
|
384
|
+
* better-sqlite3 was built with SQLITE_DEFAULT_WAL_SYNCHRONOUS=1, Node's SQLite with 2. SQLite applies that default
|
|
385
|
+
* only to a connection that opens a database already in WAL mode, read-only or not: such a connection gets
|
|
386
|
+
* synchronous = NORMAL (1) here as it did there. A connection that switches a database to WAL itself (a fresh index
|
|
387
|
+
* in openDb) keeps FULL (2), as it did there.
|
|
388
|
+
*/
|
|
389
|
+
matchWalSynchronous() {
|
|
390
|
+
const mode = this.db.prepare('PRAGMA journal_mode').get();
|
|
391
|
+
if (mode?.journal_mode === 'wal')
|
|
392
|
+
this.db.exec('PRAGMA synchronous = NORMAL');
|
|
393
|
+
}
|
|
394
|
+
}
|
|
395
|
+
const Database = SqliteDatabase;
|
|
396
|
+
exports.default = Database;
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.prepareOnce = prepareOnce;
|
|
4
|
+
exports.listStatement = listStatement;
|
|
5
|
+
exports.paddedList = paddedList;
|
|
6
|
+
exports.transactionOnce = transactionOnce;
|
|
7
|
+
/**
|
|
8
|
+
* One prepared statement per connection and SQL text, for the statements a read or a save runs again on every request.
|
|
9
|
+
*
|
|
10
|
+
* Preparing a statement costs a parse and a plan, and a `get_context` runs a few dozen small ones: preparing them again
|
|
11
|
+
* on every call was about 2 ms of its 20 (profiled with scripts/bench/mcp-latency.ts --cpu-profile). A prepared
|
|
12
|
+
* statement is reused as it is, and SQLite prepares it again by itself when the schema changes, so a migration that runs
|
|
13
|
+
* after the first use is seen. The result of a statement does not depend on how long ago it was prepared.
|
|
14
|
+
*
|
|
15
|
+
* What is shared is the statement, so what a caller leaves on it is shared too:
|
|
16
|
+
* - Do not leave a mode on it (`.pluck()`, `.raw()`, `.expand()`, `.safeIntegers()`, `.bind()`) unless every caller of
|
|
17
|
+
* that text asks for the same mode. Give a statement that is read another way its own text.
|
|
18
|
+
* - Do not cache a statement that is iterated (`.iterate()`): an open iteration holds it, and a second use fails.
|
|
19
|
+
* - Do not cache SQL whose text varies with its input (an `IN (?, ?, ...)` list of any length): each text is kept.
|
|
20
|
+
* The cache stops growing at MAX_PREPARED texts per connection; past it a statement is prepared and not kept.
|
|
21
|
+
*/
|
|
22
|
+
const MAX_PREPARED = 256;
|
|
23
|
+
const prepared = new WeakMap();
|
|
24
|
+
function prepareOnce(db, sql) {
|
|
25
|
+
let byText = prepared.get(db);
|
|
26
|
+
if (!byText)
|
|
27
|
+
prepared.set(db, byText = new Map());
|
|
28
|
+
let statement = byText.get(sql);
|
|
29
|
+
if (statement)
|
|
30
|
+
return statement;
|
|
31
|
+
statement = db.prepare(sql);
|
|
32
|
+
if (byText.size < MAX_PREPARED)
|
|
33
|
+
byText.set(sql, statement);
|
|
34
|
+
return statement;
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* The most values a list is padded to a power of two for. A longer list is prepared as it is and not kept: the padding would
|
|
38
|
+
* double it, and a list that long is not the usual one (SQLite takes 32,766 values in a statement).
|
|
39
|
+
*/
|
|
40
|
+
const MAX_PADDED_LIST = 256;
|
|
41
|
+
/** The number of values a list of `count` is padded to: the next power of two, and at least four. */
|
|
42
|
+
function paddedSize(count) {
|
|
43
|
+
return count <= 4 ? 4 : 2 ** (32 - Math.clz32(count - 1));
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* A statement over an `IN (?, ?, ...)` list whose length follows the question (the notes a read hit, the entities of those
|
|
47
|
+
* notes). The text of such a statement changes with the length, so `prepareOnce` would keep one for every length and fill its
|
|
48
|
+
* room (see the third rule above), and preparing it on every call was about a tenth of a `get_context`'s statement time (ten
|
|
49
|
+
* statements prepared again on each call). Here the list is made a power of two long, at least four, and the values are padded
|
|
50
|
+
* with `paddedList`: a NULL in an IN list matches no row, so the rows are the rows of the list itself. A connection prepares
|
|
51
|
+
* one statement for each size, a few in all, and the next call finds them ready. `build` makes the SQL from the placeholders.
|
|
52
|
+
* A list over MAX_PADDED_LIST is prepared as it is, for this call only. Use it where the list is the whole of an IN: not
|
|
53
|
+
* under NOT IN, where a NULL in the list would make the whole test unknown.
|
|
54
|
+
*/
|
|
55
|
+
function listStatement(db, count, build) {
|
|
56
|
+
if (count > MAX_PADDED_LIST)
|
|
57
|
+
return db.prepare(build('?,'.repeat(count - 1) + '?'));
|
|
58
|
+
return prepareOnce(db, build('?,'.repeat(paddedSize(count) - 1) + '?'));
|
|
59
|
+
}
|
|
60
|
+
/** The values of a list as `listStatement` binds them: padded with NULL to the size it was prepared for. */
|
|
61
|
+
function paddedList(values) {
|
|
62
|
+
const padded = values.slice();
|
|
63
|
+
if (values.length > MAX_PADDED_LIST)
|
|
64
|
+
return padded;
|
|
65
|
+
for (let size = paddedSize(values.length); padded.length < size;)
|
|
66
|
+
padded.push(null);
|
|
67
|
+
return padded;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* One transaction wrapper per connection and body, for a transaction a replay runs once per ledger event.
|
|
71
|
+
*
|
|
72
|
+
* `db.transaction(fn)` builds the wrapper's four variants (default, deferred, immediate, exclusive) every time it is called, which
|
|
73
|
+
* is about 0.05 ms: the ledger replay at startup called it twice per event, about 35 ms of the 190 ms replay of a 1,000-fact
|
|
74
|
+
* workspace. The wrapper does not depend on what the body closes over, so the body takes the connection and its inputs as
|
|
75
|
+
* arguments, is a module-level function, and is wrapped once. What runs in the transaction, and when it commits or rolls back, is
|
|
76
|
+
* unchanged; a call inside another transaction is a savepoint, as before.
|
|
77
|
+
*/
|
|
78
|
+
const transactions = new WeakMap();
|
|
79
|
+
function transactionOnce(db, body) {
|
|
80
|
+
let byBody = transactions.get(db);
|
|
81
|
+
if (!byBody)
|
|
82
|
+
transactions.set(db, byBody = new Map());
|
|
83
|
+
let tx = byBody.get(body);
|
|
84
|
+
if (!tx) {
|
|
85
|
+
tx = db.transaction((...args) => body(db, ...args));
|
|
86
|
+
byBody.set(body, tx);
|
|
87
|
+
}
|
|
88
|
+
return tx;
|
|
89
|
+
}
|