renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,109 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.tableHasColumns = tableHasColumns;
4
- exports.inReadSnapshot = inReadSnapshot;
5
- exports.databaseStamp = databaseStamp;
6
- exports.readUntilWrite = readUntilWrite;
7
- const statements_1 = require("./statements");
8
- /**
9
- * Two things a read asks of the database again and again and that change rarely: the shape of a table, and a value
10
- * derived from every row of one (the ledger's closed events). A `get_context` asked for each of them six times.
11
- */
12
- const withColumns = new WeakMap();
13
- /**
14
- * Whether `table` has every one of `columns` (the reads that predate a column ask, so an old index still answers).
15
- * Only a yes is kept for the connection: a table that lacks a column may get it from a migration, and a column that
16
- * is there is never dropped, so a no is asked again each time and a yes never is. The names are the code's own, never
17
- * a caller's.
18
- */
19
- function tableHasColumns(db, table, columns) {
20
- const key = `${table}(${columns.join(',')})`;
21
- let known = withColumns.get(db);
22
- if (known?.has(key))
23
- return true;
24
- const present = new Set(db.prepare(`PRAGMA table_info(${table})`).all().map(c => c.name));
25
- if (!columns.every(c => present.has(c)))
26
- return false;
27
- if (!known)
28
- withColumns.set(db, known = new Map());
29
- known.set(key, true);
30
- return true;
31
- }
32
- const kept = new WeakMap();
33
- /** Values kept per connection. A key may carry what a caller sent (a scope), so the oldest goes when the room is used. */
34
- const MAX_KEPT = 64;
35
- /** The connections inside a snapshot (`inReadSnapshot`). */
36
- const snapshots = new WeakSet();
37
- /**
38
- * What the database has been through: changes this connection committed (`total_changes()`, which counts every row an
39
- * INSERT, UPDATE or DELETE touched, triggers and virtual tables included) and changes any other connection committed
40
- * (`PRAGMA data_version`, which a connection's own writes do not move). Both only ever go up, so two equal stamps are
41
- * the same data. Null inside a transaction: it can still be rolled back, and its changes would be counted for good. A
42
- * snapshot is the exception: it is ended with a COMMIT and never rolled back, so what it has counted stays counted.
43
- */
44
- function stampOf(db) {
45
- if (db.inTransaction && !snapshots.has(db))
46
- return null;
47
- const version = (0, statements_1.prepareOnce)(db, 'PRAGMA data_version').pluck().get();
48
- const changes = (0, statements_1.prepareOnce)(db, 'SELECT total_changes()').pluck().get();
49
- return `${version}:${changes}`;
50
- }
51
- /**
52
- * Runs `read` in ONE transaction. A statement that runs on its own begins a read transaction and ends it again, and in WAL
53
- * mode that is two locks on the shared-memory index and a file-size call: about 5 microseconds for a lookup that takes 1.5,
54
- * and a `get_context` runs a hundred of them. Inside a transaction the lock is taken once, and every statement after the first
55
- * pays for what it reads. The reads see the database as it was when the first of them began; a commit by another process
56
- * meanwhile is in the next read, not in a mix of the two.
57
- *
58
- * `read` is for a stretch of reads that holds no `await` and writes nothing: nothing else of this process can run in a
59
- * stretch that does not yield, and a write inside it would not be durable until the transaction ended. When a transaction is
60
- * open already (a write transaction, or a snapshot outside this one) `read` just runs in it. The stamps (`databaseStamp`,
61
- * `readUntilWrite`) hold for a snapshot, which is committed and never rolled back.
62
- */
63
- function inReadSnapshot(db, read) {
64
- if (db.inTransaction)
65
- return read();
66
- (0, statements_1.prepareOnce)(db, 'BEGIN').run();
67
- snapshots.add(db);
68
- let done = false;
69
- try {
70
- const result = read();
71
- done = true;
72
- return result;
73
- }
74
- finally {
75
- snapshots.delete(db);
76
- try {
77
- (0, statements_1.prepareOnce)(db, 'COMMIT').run();
78
- }
79
- catch (err) {
80
- if (done)
81
- throw err; // the read failed first: that is the error to report
82
- }
83
- }
84
- }
85
- /** The stamp `readUntilWrite` keeps a value under. Null inside a transaction, which can still roll back. */
86
- function databaseStamp(db) {
87
- return stampOf(db);
88
- }
89
- /**
90
- * `compute()` for the database as it is now, read once and kept until anything is written to it, by this connection or
91
- * another. The same answer a fresh `compute()` gives: nothing is kept across a write, and nothing at all inside a
92
- * transaction. `compute` must read only the database, and what it returns is shared: callers must not change it.
93
- */
94
- function readUntilWrite(db, key, compute) {
95
- const stamp = stampOf(db);
96
- if (stamp === null)
97
- return compute();
98
- let byKey = kept.get(db);
99
- if (!byKey)
100
- kept.set(db, byKey = new Map());
101
- const held = byKey.get(key);
102
- if (held && held.stamp === stamp)
103
- return held.value;
104
- const value = compute();
105
- if (!byKey.has(key) && byKey.size >= MAX_KEPT)
106
- byKey.delete(byKey.keys().next().value);
107
- byKey.set(key, { stamp, value });
108
- return value;
109
- }
package/dist/db/schema.js DELETED
@@ -1,5 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.runMigrations = void 0;
4
- var migrations_1 = require("./migrations");
5
- Object.defineProperty(exports, "runMigrations", { enumerable: true, get: function () { return migrations_1.runMigrations; } });
package/dist/db/sqlite.js DELETED
@@ -1,396 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.Statement = exports.MIN_NODE = void 0;
7
- exports.nodeSupportProblem = nodeSupportProblem;
8
- exports.sqliteRuntimeProblem = sqliteRuntimeProblem;
9
- /**
10
- * The engine's SQLite driver: Node's built-in `node:sqlite`, behind the part of the better-sqlite3 API the engine uses.
11
- *
12
- * Why not better-sqlite3: it is a native addon that builds itself in an npm install script. `npx -y renor-engine` then
13
- * breaks for anyone whose npm does not run dependency scripts (`ignore-scripts=true`, newer npm defaults), and on a Node
14
- * with no prebuilt binary the install falls back to compiling C++ for over a minute while the MCP client gives up. Node
15
- * ships SQLite itself (FTS5 included) and loads sqlite-vec, whose prebuilt binary is a plain optional dependency, so the
16
- * engine installs with no script at all.
17
- *
18
- * Behaviour kept from better-sqlite3, because the engine and its tests rely on it:
19
- * - rows are plain objects and BLOBs come back as Buffer; `pluck()`, `raw()` and `safeIntegers()` work as before;
20
- * - `transaction(fn)` begins (or, nested, takes a savepoint), commits, and rolls back on a throw;
21
- * - errors carry the SQLite code name (`SQLITE_CONSTRAINT_UNIQUE`, `SQLITE_CORRUPT`, ...) in `code`;
22
- * - a 5 s busy timeout, a 16 MB page cache, and `synchronous = NORMAL` on a connection that opens a database already in
23
- * WAL mode (better-sqlite3's compiled defaults; see matchWalSynchronous);
24
- * Extension loading is allowed through the C API only (`loadExtension`, for sqlite-vec); SQL's `load_extension()` stays
25
- * refused, as it was.
26
- */
27
- const fs_1 = __importDefault(require("fs"));
28
- /** The first Node with every `node:sqlite` feature used here (extension loading, `isTransaction`, `setReturnArrays`, `backup`). */
29
- exports.MIN_NODE = '22.16.0';
30
- /** Why this Node cannot run the engine, or null when it can. */
31
- function nodeSupportProblem(version = process.versions.node) {
32
- const [major, minor] = version.split('.').map(Number);
33
- if (major >= 24 || (major === 22 && minor >= 16))
34
- return null;
35
- return `Renor needs Node ${exports.MIN_NODE} or newer (it uses Node's built-in SQLite); this is Node ${version}.`;
36
- }
37
- /**
38
- * Why the engine cannot store anything here, as one line with its fix, or null when it can: this Node has no usable
39
- * built-in SQLite, or sqlite-vec's prebuilt binary for this platform is missing or does not load. The CLI says this and
40
- * exits before it starts, so an MCP client shows the reason instead of a connection that silently failed.
41
- */
42
- function sqliteRuntimeProblem() {
43
- const node = nodeSupportProblem();
44
- if (node)
45
- return `${node} Fix: install Node 22 LTS or newer (https://nodejs.org), then run the command again.`;
46
- try {
47
- const db = new SqliteDatabase(':memory:');
48
- try {
49
- require('sqlite-vec').load(db);
50
- }
51
- finally {
52
- db.close();
53
- }
54
- }
55
- catch (err) {
56
- const reason = (err instanceof Error ? err.message.split('\n')[0] : String(err)).replace(/\.?\s*$/, '.');
57
- return `the vector search extension (sqlite-vec) did not load on ${process.platform}-${process.arch}: ${reason} `
58
- + 'Fix: let npm install optional dependencies (remove omit=optional or --no-optional from your npm settings), '
59
- + 'then run `npx -y renor-engine@latest mcp` again.';
60
- }
61
- return null;
62
- }
63
- let sqlite;
64
- function nodeSqlite() {
65
- if (sqlite)
66
- return sqlite;
67
- const problem = nodeSupportProblem();
68
- if (problem)
69
- throw new Error(`${problem} Install Node 22 LTS or newer from https://nodejs.org and run the command again.`);
70
- // Node 22 prints "SQLite is an experimental feature" once on load; to an MCP client that is noise on stderr, not news.
71
- const emitWarning = process.emitWarning;
72
- process.emitWarning = ((warning, ...rest) => {
73
- if (String(warning instanceof Error ? warning.message : warning).includes('SQLite is an experimental feature'))
74
- return;
75
- return emitWarning.call(process, warning, ...rest);
76
- });
77
- try {
78
- sqlite = require('node:sqlite');
79
- }
80
- finally {
81
- process.emitWarning = emitWarning;
82
- }
83
- return sqlite;
84
- }
85
- /** SQLite's primary result codes, by number. */
86
- const PRIMARY_CODES = ('OK ERROR INTERNAL PERM ABORT BUSY LOCKED NOMEM READONLY INTERRUPT IOERR CORRUPT NOTFOUND FULL CANTOPEN PROTOCOL EMPTY '
87
- + 'SCHEMA TOOBIG CONSTRAINT MISMATCH MISUSE NOLFS AUTH FORMAT RANGE NOTADB NOTICE WARNING').split(' ');
88
- /** The extended result codes worth telling apart; any other extended code is named by its primary code. */
89
- const EXTENDED_CODES = new Map(('261 BUSY_RECOVERY,517 BUSY_SNAPSHOT,773 BUSY_TIMEOUT,262 LOCKED_SHAREDCACHE,267 CORRUPT_VTAB,523 CORRUPT_SEQUENCE,'
90
- + '779 CORRUPT_INDEX,264 READONLY_RECOVERY,520 READONLY_CANTLOCK,776 READONLY_ROLLBACK,1032 READONLY_DBMOVED,'
91
- + '275 CONSTRAINT_CHECK,531 CONSTRAINT_COMMITHOOK,787 CONSTRAINT_FOREIGNKEY,1043 CONSTRAINT_FUNCTION,'
92
- + '1299 CONSTRAINT_NOTNULL,1555 CONSTRAINT_PRIMARYKEY,1811 CONSTRAINT_TRIGGER,2067 CONSTRAINT_UNIQUE,'
93
- + '2323 CONSTRAINT_VTAB,2579 CONSTRAINT_ROWID,3091 CONSTRAINT_DATATYPE').split(',').map(pair => [Number(pair.split(' ')[0]), pair.split(' ')[1]]));
94
- /** Gives a `node:sqlite` error better-sqlite3's `code` (`SQLITE_<NAME>`); every other error passes through unchanged. */
95
- function withSqliteCode(err) {
96
- const e = err;
97
- if (e && e.code === 'ERR_SQLITE_ERROR' && typeof e.errcode === 'number') {
98
- const name = EXTENDED_CODES.get(e.errcode) ?? PRIMARY_CODES[e.errcode & 0xff] ?? 'ERROR';
99
- try {
100
- e.code = `SQLITE_${name}`;
101
- }
102
- catch { /* a frozen error keeps its code */ }
103
- }
104
- return err;
105
- }
106
- function guarded(work) {
107
- try {
108
- return work();
109
- }
110
- catch (err) {
111
- throw withSqliteCode(err);
112
- }
113
- }
114
- /** Every bigint in a row (or rows) as the nearest double, as better-sqlite3 read an integer beyond 2^53. */
115
- function roundBigInts(value) {
116
- if (typeof value === 'bigint')
117
- return Number(value);
118
- if (Array.isArray(value))
119
- return value.map(roundBigInts);
120
- if (value !== null && typeof value === 'object' && !ArrayBuffer.isView(value)) {
121
- const out = {};
122
- for (const key in value)
123
- out[key] = roundBigInts(value[key]);
124
- return out;
125
- }
126
- return value;
127
- }
128
- function asBuffer(value) {
129
- return value instanceof Uint8Array && !Buffer.isBuffer(value) ? Buffer.from(value.buffer, value.byteOffset, value.byteLength) : value;
130
- }
131
- /** better-sqlite3 takes named parameters as an object in any position and positional ones also as arrays. */
132
- function bindArgs(params) {
133
- let named;
134
- const positional = [];
135
- for (const p of params) {
136
- if (Array.isArray(p))
137
- positional.push(...p);
138
- else if (p !== null && typeof p === 'object' && !ArrayBuffer.isView(p))
139
- named = p;
140
- else
141
- positional.push(p);
142
- }
143
- return named === undefined ? positional : [named, ...positional];
144
- }
145
- /** SQL that can change a table's columns, after which this connection's prepared statements are prepared again. */
146
- const SCHEMA_CHANGE = /\b(ALTER|CREATE|DROP)\b/i;
147
- const WRITE_OPCODES = new Set(['JournalMode', 'Vacuum', 'VCreate', 'VDestroy', 'VUpdate', 'ParseSchema']);
148
- class Statement {
149
- database;
150
- stmt;
151
- source;
152
- mode = 'object';
153
- bigInts = false;
154
- changesSchema;
155
- generation;
156
- constructor(database, stmt, source) {
157
- this.database = database;
158
- this.stmt = stmt;
159
- this.source = source;
160
- this.changesSchema = SCHEMA_CHANGE.test(source);
161
- this.generation = database.schemaGeneration;
162
- }
163
- run(...params) {
164
- const result = guarded(() => this.current().run(...bindArgs(params)));
165
- if (this.changesSchema)
166
- this.database.schemaChanged();
167
- // `changes` is a number even under safeIntegers(), as in better-sqlite3; only lastInsertRowid becomes a bigint.
168
- return { changes: Number(result.changes), lastInsertRowid: result.lastInsertRowid };
169
- }
170
- get(...params) {
171
- const row = this.lossy(() => this.current().get(...bindArgs(params)));
172
- return row === undefined ? undefined : this.shape(row);
173
- }
174
- all(...params) {
175
- return this.lossy(() => this.current().all(...bindArgs(params))).map(row => this.shape(row));
176
- }
177
- /**
178
- * Errors raised while stepping get the same `SQLITE_*` code as the other calls. An integer beyond 2^53 still throws
179
- * RangeError here (better-sqlite3 returned it rounded); use safeIntegers(), or get()/all(), which round as before.
180
- */
181
- *iterate(...params) {
182
- const rows = guarded(() => this.current().iterate(...bindArgs(params)));
183
- try {
184
- for (;;) {
185
- const step = guarded(() => rows.next());
186
- if (step.done)
187
- return;
188
- yield this.shape(step.value);
189
- }
190
- }
191
- finally {
192
- rows.return?.();
193
- }
194
- }
195
- pluck(toggleState = true) {
196
- return this.setMode(toggleState ? 'pluck' : this.mode === 'pluck' ? 'object' : this.mode);
197
- }
198
- raw(toggleState = true) {
199
- return this.setMode(toggleState ? 'raw' : this.mode === 'raw' ? 'object' : this.mode);
200
- }
201
- safeIntegers(toggleState = true) {
202
- this.bigInts = toggleState;
203
- this.stmt.setReadBigInts(toggleState);
204
- return this;
205
- }
206
- /**
207
- * Whether the statement writes nothing, as SQLite's sqlite3_stmt_readonly() decides it: its program opens no write
208
- * transaction and changes no journal mode, virtual table or file. Read from the statement's EXPLAIN on demand.
209
- */
210
- get readonly() {
211
- const program = guarded(() => this.database.prepareNative(`EXPLAIN ${this.source}`).all());
212
- return !program.some(op => (op.opcode === 'Transaction' && op.p2 > 0) || WRITE_OPCODES.has(op.opcode));
213
- }
214
- columns() {
215
- return this.current().columns();
216
- }
217
- /**
218
- * better-sqlite3 returned an integer beyond 2^53 rounded to the nearest double; node:sqlite throws RangeError
219
- * (ERR_OUT_OF_RANGE) instead. A statement that writes nothing is read again as bigints and rounded the same way; a
220
- * statement that writes is never run twice, so its RangeError stands.
221
- */
222
- lossy(read) {
223
- try {
224
- return guarded(read);
225
- }
226
- catch (err) {
227
- if (this.bigInts || err.code !== 'ERR_OUT_OF_RANGE' || !this.readonly)
228
- throw err;
229
- this.stmt.setReadBigInts(true);
230
- try {
231
- return roundBigInts(guarded(read));
232
- }
233
- finally {
234
- this.stmt.setReadBigInts(false);
235
- }
236
- }
237
- }
238
- /**
239
- * The native statement, prepared again after this connection changed the schema. SQLite re-prepares a statement by
240
- * itself, but `node:sqlite` reads the result's columns before that happens, so the first run after an ALTER TABLE
241
- * would return the old columns; better-sqlite3 did not.
242
- */
243
- current() {
244
- if (this.generation !== this.database.schemaGeneration) {
245
- this.stmt = this.database.prepareNative(this.source);
246
- this.stmt.setReturnArrays(this.mode !== 'object');
247
- this.stmt.setReadBigInts(this.bigInts);
248
- this.generation = this.database.schemaGeneration;
249
- }
250
- return this.stmt;
251
- }
252
- setMode(mode) {
253
- this.mode = mode;
254
- this.stmt.setReturnArrays(mode !== 'object');
255
- return this;
256
- }
257
- shape(row) {
258
- if (this.mode === 'pluck')
259
- return asBuffer(row[0]);
260
- if (this.mode === 'raw')
261
- return row.map(asBuffer);
262
- const out = {};
263
- for (const key in row)
264
- out[key] = asBuffer(row[key]);
265
- return out;
266
- }
267
- }
268
- exports.Statement = Statement;
269
- class SqliteDatabase {
270
- name;
271
- readonly;
272
- memory;
273
- db;
274
- closed = false;
275
- extensions = new Set();
276
- /** @internal Bumped whenever this connection may have changed the schema; see Statement's `current()`. */
277
- schemaGeneration = 0;
278
- constructor(filename = '', options = {}) {
279
- this.name = filename;
280
- this.readonly = options.readonly === true;
281
- this.memory = filename === ':memory:' || filename === '';
282
- if (options.fileMustExist && !this.memory && !fs_1.default.existsSync(filename)) {
283
- throw Object.assign(new Error(`unable to open database file: ${filename}`), { code: 'SQLITE_CANTOPEN' });
284
- }
285
- const { DatabaseSync } = nodeSqlite();
286
- this.db = guarded(() => new DatabaseSync(filename, { readOnly: this.readonly, allowExtension: true }));
287
- try {
288
- this.db.exec(`PRAGMA busy_timeout = ${Math.max(0, Math.trunc(options.timeout ?? 5000))}`);
289
- this.db.exec('PRAGMA cache_size = -16000');
290
- this.matchWalSynchronous();
291
- }
292
- catch (err) {
293
- this.db.close();
294
- throw withSqliteCode(err);
295
- }
296
- }
297
- get open() {
298
- return !this.closed;
299
- }
300
- get inTransaction() {
301
- return !this.closed && this.db.isTransaction;
302
- }
303
- prepare(source) {
304
- return new Statement(this, guarded(() => this.db.prepare(source)), source);
305
- }
306
- /** @internal The `node:sqlite` statement itself, for Statement's own bookkeeping. */
307
- prepareNative(source) {
308
- return this.db.prepare(source);
309
- }
310
- exec(source) {
311
- guarded(() => this.db.exec(source));
312
- if (SCHEMA_CHANGE.test(source))
313
- this.schemaChanged();
314
- return this;
315
- }
316
- /** @internal */
317
- schemaChanged() {
318
- this.schemaGeneration++;
319
- }
320
- pragma(source, options = {}) {
321
- const stmt = guarded(() => this.db.prepare(`PRAGMA ${source}`));
322
- if (options.simple) {
323
- stmt.setReturnArrays(true);
324
- const row = guarded(() => stmt.get());
325
- return row === undefined ? undefined : asBuffer(row[0]);
326
- }
327
- return guarded(() => stmt.all()).map(row => ({ ...row }));
328
- }
329
- transaction(fn) {
330
- const wrap = (begin) => {
331
- const db = this;
332
- return function sqliteTransaction(...args) {
333
- const nested = db.inTransaction;
334
- db.exec(nested ? 'SAVEPOINT "_renor_tx"' : begin);
335
- try {
336
- const result = fn.apply(this, args);
337
- if (result && typeof result.then === 'function') {
338
- throw new TypeError('Transaction function cannot return a promise');
339
- }
340
- db.exec(nested ? 'RELEASE "_renor_tx"' : 'COMMIT');
341
- return result;
342
- }
343
- catch (err) {
344
- if (db.inTransaction) {
345
- if (nested)
346
- db.exec('ROLLBACK TO "_renor_tx"; RELEASE "_renor_tx"');
347
- else
348
- db.exec('ROLLBACK');
349
- }
350
- throw err;
351
- }
352
- };
353
- };
354
- const tx = wrap('BEGIN');
355
- tx.default = tx;
356
- tx.deferred = wrap('BEGIN DEFERRED');
357
- tx.immediate = wrap('BEGIN IMMEDIATE');
358
- tx.exclusive = wrap('BEGIN EXCLUSIVE');
359
- return tx;
360
- }
361
- /**
362
- * Loads an extension once per connection: a second load only registers the same functions again, and that fails
363
- * ("error during initialization") inside a transaction that has deleted rows behind a virtual table, where
364
- * src/indexer/outsideRoot.ts loads sqlite-vec.
365
- */
366
- loadExtension(path) {
367
- if (this.extensions.has(path))
368
- return this;
369
- guarded(() => this.db.loadExtension(path));
370
- this.extensions.add(path);
371
- return this;
372
- }
373
- backup(destinationFile) {
374
- return nodeSqlite().backup(this.db, destinationFile);
375
- }
376
- close() {
377
- if (!this.closed) {
378
- this.closed = true;
379
- this.db.close();
380
- }
381
- return this;
382
- }
383
- /**
384
- * better-sqlite3 was built with SQLITE_DEFAULT_WAL_SYNCHRONOUS=1, Node's SQLite with 2. SQLite applies that default
385
- * only to a connection that opens a database already in WAL mode, read-only or not: such a connection gets
386
- * synchronous = NORMAL (1) here as it did there. A connection that switches a database to WAL itself (a fresh index
387
- * in openDb) keeps FULL (2), as it did there.
388
- */
389
- matchWalSynchronous() {
390
- const mode = this.db.prepare('PRAGMA journal_mode').get();
391
- if (mode?.journal_mode === 'wal')
392
- this.db.exec('PRAGMA synchronous = NORMAL');
393
- }
394
- }
395
- const Database = SqliteDatabase;
396
- exports.default = Database;
@@ -1,89 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.prepareOnce = prepareOnce;
4
- exports.listStatement = listStatement;
5
- exports.paddedList = paddedList;
6
- exports.transactionOnce = transactionOnce;
7
- /**
8
- * One prepared statement per connection and SQL text, for the statements a read or a save runs again on every request.
9
- *
10
- * Preparing a statement costs a parse and a plan, and a `get_context` runs a few dozen small ones: preparing them again
11
- * on every call was about 2 ms of its 20 (profiled with scripts/bench/mcp-latency.ts --cpu-profile). A prepared
12
- * statement is reused as it is, and SQLite prepares it again by itself when the schema changes, so a migration that runs
13
- * after the first use is seen. The result of a statement does not depend on how long ago it was prepared.
14
- *
15
- * What is shared is the statement, so what a caller leaves on it is shared too:
16
- * - Do not leave a mode on it (`.pluck()`, `.raw()`, `.expand()`, `.safeIntegers()`, `.bind()`) unless every caller of
17
- * that text asks for the same mode. Give a statement that is read another way its own text.
18
- * - Do not cache a statement that is iterated (`.iterate()`): an open iteration holds it, and a second use fails.
19
- * - Do not cache SQL whose text varies with its input (an `IN (?, ?, ...)` list of any length): each text is kept.
20
- * The cache stops growing at MAX_PREPARED texts per connection; past it a statement is prepared and not kept.
21
- */
22
- const MAX_PREPARED = 256;
23
- const prepared = new WeakMap();
24
- function prepareOnce(db, sql) {
25
- let byText = prepared.get(db);
26
- if (!byText)
27
- prepared.set(db, byText = new Map());
28
- let statement = byText.get(sql);
29
- if (statement)
30
- return statement;
31
- statement = db.prepare(sql);
32
- if (byText.size < MAX_PREPARED)
33
- byText.set(sql, statement);
34
- return statement;
35
- }
36
- /**
37
- * The most values a list is padded to a power of two for. A longer list is prepared as it is and not kept: the padding would
38
- * double it, and a list that long is not the usual one (SQLite takes 32,766 values in a statement).
39
- */
40
- const MAX_PADDED_LIST = 256;
41
- /** The number of values a list of `count` is padded to: the next power of two, and at least four. */
42
- function paddedSize(count) {
43
- return count <= 4 ? 4 : 2 ** (32 - Math.clz32(count - 1));
44
- }
45
- /**
46
- * A statement over an `IN (?, ?, ...)` list whose length follows the question (the notes a read hit, the entities of those
47
- * notes). The text of such a statement changes with the length, so `prepareOnce` would keep one for every length and fill its
48
- * room (see the third rule above), and preparing it on every call was about a tenth of a `get_context`'s statement time (ten
49
- * statements prepared again on each call). Here the list is made a power of two long, at least four, and the values are padded
50
- * with `paddedList`: a NULL in an IN list matches no row, so the rows are the rows of the list itself. A connection prepares
51
- * one statement for each size, a few in all, and the next call finds them ready. `build` makes the SQL from the placeholders.
52
- * A list over MAX_PADDED_LIST is prepared as it is, for this call only. Use it where the list is the whole of an IN: not
53
- * under NOT IN, where a NULL in the list would make the whole test unknown.
54
- */
55
- function listStatement(db, count, build) {
56
- if (count > MAX_PADDED_LIST)
57
- return db.prepare(build('?,'.repeat(count - 1) + '?'));
58
- return prepareOnce(db, build('?,'.repeat(paddedSize(count) - 1) + '?'));
59
- }
60
- /** The values of a list as `listStatement` binds them: padded with NULL to the size it was prepared for. */
61
- function paddedList(values) {
62
- const padded = values.slice();
63
- if (values.length > MAX_PADDED_LIST)
64
- return padded;
65
- for (let size = paddedSize(values.length); padded.length < size;)
66
- padded.push(null);
67
- return padded;
68
- }
69
- /**
70
- * One transaction wrapper per connection and body, for a transaction a replay runs once per ledger event.
71
- *
72
- * `db.transaction(fn)` builds the wrapper's four variants (default, deferred, immediate, exclusive) every time it is called, which
73
- * is about 0.05 ms: the ledger replay at startup called it twice per event, about 35 ms of the 190 ms replay of a 1,000-fact
74
- * workspace. The wrapper does not depend on what the body closes over, so the body takes the connection and its inputs as
75
- * arguments, is a module-level function, and is wrapped once. What runs in the transaction, and when it commits or rolls back, is
76
- * unchanged; a call inside another transaction is a savepoint, as before.
77
- */
78
- const transactions = new WeakMap();
79
- function transactionOnce(db, body) {
80
- let byBody = transactions.get(db);
81
- if (!byBody)
82
- transactions.set(db, byBody = new Map());
83
- let tx = byBody.get(body);
84
- if (!tx) {
85
- tx = db.transaction((...args) => body(db, ...args));
86
- byBody.set(body, tx);
87
- }
88
- return tx;
89
- }