renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,674 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.VectorIndex = exports.scanSlotsL2 = exports.scanPackedL2 = exports.KNN_MAX_K = void 0;
4
- exports.kthSmallest = kthSmallest;
5
- exports.markVectorsUnfinished = markVectorsUnfinished;
6
- const simdScan_1 = require("./simdScan");
7
- // eslint-disable-next-line @typescript-eslint/no-var-requires
8
- const sqliteVec = require('sqlite-vec');
9
- /** sqlite-vec's ceiling on `k` in one nearest-neighbour query. */
10
- exports.KNN_MAX_K = 4096;
11
- /**
12
- * Above this many bytes of vectors the process keeps asking sqlite-vec, instead of keeping a second copy of every
13
- * vector. 64 MiB is about 21,000 vectors at the 768 dimensions a hosted workspace uses, and the measured fixture
14
- * (about 10,000) fits under it.
15
- */
16
- const MIRROR_MAX_BYTES = 64 * 1024 * 1024;
17
- /**
18
- * How far past the k-th scan distance a vector is still rescored exactly. The scan is ordinary f64 arithmetic and is
19
- * not sqlite-vec's distance: on 768-d unit vectors, hash and dense, the largest gap seen against `vec_distance_l2`
20
- * was under 1e-6, and 1e-3 still added only a few extra rows. A wider gap is slower, not wrong. A row whose exact
21
- * distance then disagrees by more than half of this is a broken bound, and that one query uses the KNN path.
22
- */
23
- const MIRROR_SLACK = 1e-3;
24
- /** Past this many exact distances, one KNN query is the cheaper exact answer (a tie across the whole index). */
25
- const MIRROR_EXACT_LIMIT = 256;
26
- /** The most nearest vectors a search takes the k-th distance of with a heap; a larger k sorts a copy. */
27
- const KTH_HEAP_MAX = 64;
28
- const kthHeap = new Float64Array(KTH_HEAP_MAX);
29
- /**
30
- * The k-th smallest of `values[0..n)` (1 <= k <= n): what sorting a copy and reading index k - 1 gives, without the
31
- * copy and the sort (about 0.8 ms for 9,661 distances, against 0.03 ms for this). A sorted copy puts a NaN (a stored
32
- * vector that is not a number, or a query that is not) after every number, so a NaN is never among the k smallest, and
33
- * the k-th is a NaN only when fewer than k of the values are numbers; the heap skips NaNs and answers the same.
34
- */
35
- function kthSmallest(values, n, k) {
36
- if (k > KTH_HEAP_MAX) {
37
- const sorted = values.slice(0, n);
38
- sorted.sort();
39
- return sorted[k - 1];
40
- }
41
- const heap = kthHeap; // the k smallest numbers seen so far, the largest of them at heap[0]
42
- let size = 0;
43
- for (let i = 0; i < n; i++) {
44
- const v = values[i];
45
- if (v !== v)
46
- continue; // NaN
47
- if (size < k) {
48
- let at = size++;
49
- while (at > 0) {
50
- const parent = (at - 1) >> 1;
51
- if (heap[parent] >= v)
52
- break;
53
- heap[at] = heap[parent];
54
- at = parent;
55
- }
56
- heap[at] = v;
57
- }
58
- else if (v < heap[0]) {
59
- let at = 0;
60
- for (;;) {
61
- let child = 2 * at + 1;
62
- if (child >= k)
63
- break;
64
- if (child + 1 < k && heap[child + 1] > heap[child])
65
- child++;
66
- if (heap[child] <= v)
67
- break;
68
- heap[at] = heap[child];
69
- at = child;
70
- }
71
- heap[at] = v;
72
- }
73
- }
74
- return size < k ? Number.NaN : heap[0];
75
- }
76
- /**
77
- * The scan is built at load, not written as a function the caller can inline. Inlined into `searchMirror`, it
78
- * deoptimizes with that method: a scoped search and an unscoped one disagree about the shape of the slot list, the
79
- * method bails out, and the scan then runs in the interpreter. On the 9,661 × 768 fixture that is about 40ms instead
80
- * of about 7ms, on some calls and not others, which is what puts the median on the slow ones. The distances only
81
- * choose the rows to rescore; the score is still `vec_distance_l2`.
82
- */
83
- function compileScan(params, body) {
84
- return new Function(...params.split(','), `'use strict';\n${body}`);
85
- }
86
- exports.scanPackedL2 = compileScan('data,count,dims,q,out', `
87
- let i = 0;
88
- if ((dims & 7) === 0) {
89
- // Four rows at a time, each summed from the first dimension up in its own chain: the query is read once for the four,
90
- // and the four chains do not wait on one another (about a fifth less time than one row at a time).
91
- for (; i + 3 < count; i += 4) {
92
- const b0 = i * dims, b1 = b0 + dims, b2 = b1 + dims, b3 = b2 + dims;
93
- let s0 = 0, s1 = 0, s2 = 0, s3 = 0;
94
- for (let d = 0; d < dims; d += 2) {
95
- const qd = q[d], qe = q[d + 1];
96
- const x0 = data[b0 + d] - qd, x1 = data[b1 + d] - qd, x2 = data[b2 + d] - qd, x3 = data[b3 + d] - qd;
97
- const y0 = data[b0 + d + 1] - qe, y1 = data[b1 + d + 1] - qe, y2 = data[b2 + d + 1] - qe, y3 = data[b3 + d + 1] - qe;
98
- s0 += x0 * x0; s1 += x1 * x1; s2 += x2 * x2; s3 += x3 * x3;
99
- s0 += y0 * y0; s1 += y1 * y1; s2 += y2 * y2; s3 += y3 * y3;
100
- }
101
- out[i] = Math.sqrt(s0); out[i + 1] = Math.sqrt(s1); out[i + 2] = Math.sqrt(s2); out[i + 3] = Math.sqrt(s3);
102
- }
103
- for (; i < count; i++) {
104
- const base = i * dims;
105
- let sum = 0;
106
- for (let d = 0; d < dims; d += 8) {
107
- const a0 = data[base + d] - q[d];
108
- const a1 = data[base + d + 1] - q[d + 1];
109
- const a2 = data[base + d + 2] - q[d + 2];
110
- const a3 = data[base + d + 3] - q[d + 3];
111
- const a4 = data[base + d + 4] - q[d + 4];
112
- const a5 = data[base + d + 5] - q[d + 5];
113
- const a6 = data[base + d + 6] - q[d + 6];
114
- const a7 = data[base + d + 7] - q[d + 7];
115
- sum += a0 * a0 + a1 * a1 + a2 * a2 + a3 * a3 + a4 * a4 + a5 * a5 + a6 * a6 + a7 * a7;
116
- }
117
- out[i] = Math.sqrt(sum);
118
- }
119
- return;
120
- }
121
- for (; i < count; i++) {
122
- const base = i * dims;
123
- let sum = 0;
124
- for (let d = 0; d < dims; d++) {
125
- const diff = data[base + d] - q[d];
126
- sum += diff * diff;
127
- }
128
- out[i] = Math.sqrt(sum);
129
- }
130
- `);
131
- /** Same scan over an arbitrary slot list (a scope), not the packed prefix. */
132
- exports.scanSlotsL2 = compileScan('data,slots,dims,q,out', `
133
- const n = slots.length;
134
- let i = 0;
135
- if ((dims & 7) === 0) {
136
- for (; i + 3 < n; i += 4) {
137
- const b0 = slots[i] * dims, b1 = slots[i + 1] * dims, b2 = slots[i + 2] * dims, b3 = slots[i + 3] * dims;
138
- let s0 = 0, s1 = 0, s2 = 0, s3 = 0;
139
- for (let d = 0; d < dims; d += 2) {
140
- const qd = q[d], qe = q[d + 1];
141
- const x0 = data[b0 + d] - qd, x1 = data[b1 + d] - qd, x2 = data[b2 + d] - qd, x3 = data[b3 + d] - qd;
142
- const y0 = data[b0 + d + 1] - qe, y1 = data[b1 + d + 1] - qe, y2 = data[b2 + d + 1] - qe, y3 = data[b3 + d + 1] - qe;
143
- s0 += x0 * x0; s1 += x1 * x1; s2 += x2 * x2; s3 += x3 * x3;
144
- s0 += y0 * y0; s1 += y1 * y1; s2 += y2 * y2; s3 += y3 * y3;
145
- }
146
- out[i] = Math.sqrt(s0); out[i + 1] = Math.sqrt(s1); out[i + 2] = Math.sqrt(s2); out[i + 3] = Math.sqrt(s3);
147
- }
148
- for (; i < n; i++) {
149
- const base = slots[i] * dims;
150
- let sum = 0;
151
- for (let d = 0; d < dims; d += 8) {
152
- const a0 = data[base + d] - q[d];
153
- const a1 = data[base + d + 1] - q[d + 1];
154
- const a2 = data[base + d + 2] - q[d + 2];
155
- const a3 = data[base + d + 3] - q[d + 3];
156
- const a4 = data[base + d + 4] - q[d + 4];
157
- const a5 = data[base + d + 5] - q[d + 5];
158
- const a6 = data[base + d + 6] - q[d + 6];
159
- const a7 = data[base + d + 7] - q[d + 7];
160
- sum += a0 * a0 + a1 * a1 + a2 * a2 + a3 * a3 + a4 * a4 + a5 * a5 + a6 * a6 + a7 * a7;
161
- }
162
- out[i] = Math.sqrt(sum);
163
- }
164
- return;
165
- }
166
- for (; i < n; i++) {
167
- const base = slots[i] * dims;
168
- let sum = 0;
169
- for (let d = 0; d < dims; d++) {
170
- const diff = data[base + d] - q[d];
171
- sum += diff * diff;
172
- }
173
- out[i] = Math.sqrt(sum);
174
- }
175
- `);
176
- /**
177
- * For a writer that gives the chunks it writes no vector (a process with no embedding provider, or an import): when the
178
- * index records the space its vectors are in, mark it unfinished (see `VectorIndex.beginRebuild`), so the next process
179
- * with that model embeds the chunks written meanwhile before semantic search reads the index again. An index that
180
- * records no space needs no mark: one with vectors is rebuilt anyway and one without is completed (see
181
- * `claimEmbeddingSpace` in `mvp/engine.ts`). Plain SQL: it does not need the vector extension. True when this call set
182
- * the mark.
183
- */
184
- function markVectorsUnfinished(db) {
185
- const meta = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'chunk_embedding_meta'").get();
186
- if (!meta || db.prepare("SELECT 1 FROM chunk_embedding_meta WHERE key = 'space'").get() === undefined)
187
- return false;
188
- return db.prepare("INSERT INTO chunk_embedding_meta(key, value) VALUES('rebuilding', '1') ON CONFLICT(key) DO NOTHING").run().changes > 0;
189
- }
190
- /** The insert found a vector already stored under this chunk id: the only constraint the insert can hit. */
191
- function storedAlready(err) {
192
- return err instanceof Error && /UNIQUE constraint failed/.test(err.message);
193
- }
194
- class VectorIndex {
195
- db;
196
- dimensions;
197
- statements = new Map();
198
- /** Exact float32 copy of the stored vectors, or null when this process has not built one (or the index is past the cap). */
199
- mirror = null;
200
- /** The scan's distances for one search, kept for the next so a search allocates nothing for them. */
201
- scanned = new Float64Array(0);
202
- constructor(db, dimensions) {
203
- this.db = db;
204
- this.dimensions = dimensions;
205
- // Load sqlite-vec extension
206
- sqliteVec.load(db);
207
- }
208
- initTable() {
209
- const existed = this.db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'chunk_embeddings'").get() !== undefined;
210
- this.db.prepare(`CREATE VIRTUAL TABLE IF NOT EXISTS chunk_embeddings USING vec0(embedding FLOAT[${this.dimensions}])`).run();
211
- this.ensureMeta();
212
- // A table created now holds no vectors, so what was recorded for an earlier one (a rebuild drops it) no
213
- // longer describes anything.
214
- if (!existed)
215
- this.db.prepare("DELETE FROM chunk_embedding_meta WHERE key IN ('space', 'rebuilding')").run();
216
- }
217
- metaReady = false;
218
- ensureMeta() {
219
- if (this.metaReady)
220
- return;
221
- this.db.prepare('CREATE TABLE IF NOT EXISTS chunk_embedding_meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)').run();
222
- this.metaReady = true;
223
- }
224
- /**
225
- * The embedding space (see embeddingSpace) every stored vector is in, as recorded when the index first held
226
- * vectors; null when none was recorded (a new index, or one built before spaces were recorded).
227
- */
228
- space() {
229
- this.ensureMeta();
230
- const row = this.stmt("SELECT value FROM chunk_embedding_meta WHERE key = 'space'").get();
231
- return row?.value ?? null;
232
- }
233
- setSpace(space) {
234
- this.ensureMeta();
235
- this.stmt("INSERT INTO chunk_embedding_meta(key, value) VALUES('space', ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value").run(space);
236
- }
237
- hasVectors() {
238
- return this.stmt('SELECT 1 FROM chunk_embeddings LIMIT 1').get() !== undefined;
239
- }
240
- /** Whether the vectors are being written afresh (see beginRebuild) and some are still to come. */
241
- rebuilding() {
242
- this.ensureMeta();
243
- return this.stmt("SELECT 1 FROM chunk_embedding_meta WHERE key = 'rebuilding'").get() !== undefined;
244
- }
245
- /**
246
- * Start the vectors over in `space`: delete every one, record `space` as the space every vector written from now
247
- * on is in, and mark the index unfinished, in one step, so a crash leaves the index as it was or named, empty and
248
- * unfinished. A stored vector cannot say which model wrote it, so the only index whose vectors can be vouched
249
- * for is one written from empty.
250
- */
251
- beginRebuild(space) {
252
- this.ensureMeta();
253
- this.db.transaction(() => {
254
- this.stmt('DELETE FROM chunk_embeddings').run();
255
- this.setSpace(space);
256
- this.stmt("INSERT INTO chunk_embedding_meta(key, value) VALUES('rebuilding', '1') ON CONFLICT(key) DO NOTHING").run();
257
- })();
258
- this.mirror = { ids: [], slot: new Map(), data: new Float32Array(0), rows: null, simdTried: false, count: 0, cap: 0, version: this.dataVersion() };
259
- }
260
- /** Every vector the rebuild had to write is written: the index is whole again. */
261
- finishRebuild() {
262
- this.ensureMeta();
263
- this.stmt("DELETE FROM chunk_embedding_meta WHERE key = 'rebuilding'").run();
264
- }
265
- /**
266
- * Preparing a statement on the vec0 table costs about ten times what running it does, and indexing
267
- * runs a few per chunk, so each is prepared once, on first use, and kept.
268
- */
269
- stmt(sql) {
270
- let statement = this.statements.get(sql);
271
- if (!statement) {
272
- statement = this.db.prepare(sql);
273
- this.statements.set(sql, statement);
274
- }
275
- return statement;
276
- }
277
- normalize(vec) {
278
- const mag = Math.sqrt(vec.reduce((sum, v) => sum + v * v, 0));
279
- if (mag === 0)
280
- return vec;
281
- return vec.map(v => v / mag);
282
- }
283
- upsert(chunkId, embedding) {
284
- const floats = new Float32Array(this.normalize(embedding));
285
- const buf = Buffer.from(floats.buffer);
286
- this.stmt('DELETE FROM chunk_embeddings WHERE rowid = ?').run(BigInt(chunkId));
287
- this.stmt('INSERT INTO chunk_embeddings(rowid, embedding) VALUES(?, ?)').run(BigInt(chunkId), buf);
288
- this.remember(chunkId, floats);
289
- }
290
- /**
291
- * The vector of a chunk this pass has just written. Chunk ids are never reused (AUTOINCREMENT), so such a chunk usually
292
- * has no vector to replace, and the delete `upsert` runs first would find nothing: this skips it. A background job
293
- * (rebuild.ts takes no lock) can store a vector for the chunk while the pass is embedding. The insert then finds the row,
294
- * and the chunk is upserted, as it was before this method existed, so the rows are the same in every case.
295
- */
296
- insertNew(chunkId, embedding) {
297
- const floats = new Float32Array(this.normalize(embedding));
298
- const buf = Buffer.from(floats.buffer);
299
- try {
300
- this.stmt('INSERT INTO chunk_embeddings(rowid, embedding) VALUES(?, ?)').run(BigInt(chunkId), buf);
301
- }
302
- catch (err) {
303
- if (!storedAlready(err))
304
- throw err;
305
- this.upsert(chunkId, embedding);
306
- return;
307
- }
308
- this.remember(chunkId, floats);
309
- }
310
- /**
311
- * Give chunk `toId` the vector stored for chunk `fromId`, byte for byte, without embedding anything.
312
- * Returns false (and leaves `toId` without a vector) when `fromId` has none.
313
- */
314
- copy(fromId, toId) {
315
- if (fromId === toId)
316
- return this.get(fromId) !== null;
317
- const dims = this.dimensions;
318
- const src = this.mirror?.slot.get(fromId);
319
- const floats = src === undefined || !this.mirror
320
- ? null
321
- : this.mirror.data.slice(src * dims, (src + 1) * dims);
322
- this.stmt('DELETE FROM chunk_embeddings WHERE rowid = ?').run(BigInt(toId));
323
- const copied = this.stmt('INSERT INTO chunk_embeddings(rowid, embedding) SELECT ?, embedding FROM chunk_embeddings WHERE rowid = ?').run(BigInt(toId), BigInt(fromId)).changes > 0;
324
- if (this.mirror) {
325
- // Replacing an existing vector does not change the row count, so the copy has to be written into the mirror.
326
- // A source this mirror does not hold means it has already drifted: drop it and let the next search reload.
327
- if (copied && floats) {
328
- this.forget(toId);
329
- this.remember(toId, floats);
330
- }
331
- else if (!copied)
332
- this.forget(toId);
333
- else
334
- this.mirror = null;
335
- }
336
- return copied;
337
- }
338
- /**
339
- * How many of these chunks have a stored vector. Asked of the table's rowid shadow table, which finds each one by
340
- * key (as `chunksWithoutVectors` does): the vec0 table itself reads every stored vector to answer an IN list, so
341
- * the count cost what the index holds rather than how many were asked about.
342
- */
343
- countStored(chunkIds) {
344
- const row = this.stmt('SELECT COUNT(*) AS n FROM chunk_embeddings_rowids WHERE rowid IN (SELECT value FROM json_each(?))').get(JSON.stringify(chunkIds));
345
- return row.n;
346
- }
347
- get(chunkId) {
348
- const row = this.stmt('SELECT embedding FROM chunk_embeddings WHERE rowid = ?').get(BigInt(chunkId));
349
- if (!row)
350
- return null;
351
- return Array.from(new Float32Array(row.embedding.buffer ?? row.embedding));
352
- }
353
- /**
354
- * The `topK` nearest vectors, nearest first, and with them every other vector exactly as near as the farthest of
355
- * those. Of several equally near vectors a nearest-neighbour query picks by where they are stored, and vectors are
356
- * stored in the order indexing wrote them, which concurrent indexing does not fix; so a tie at the edge is returned
357
- * whole, for the caller to cut by what the chunks are (semanticSearch). Scores are sqlite-vec's own distances
358
- * (`1 - d²/2`), never the in-process scan's, so a reply is the one the KNN query would have built.
359
- *
360
- * Given `among` (chunk ids), only those vectors are candidates: the nearest of them, by the same rule.
361
- */
362
- search(query, topK, among) {
363
- if (topK < 1)
364
- return [];
365
- const qf = new Float32Array(this.normalize(query));
366
- const buf = Buffer.from(qf.buffer, qf.byteOffset, qf.byteLength);
367
- if (this.mirrorReady()) {
368
- const fast = this.searchMirror(qf, buf, topK, among);
369
- if (fast !== null)
370
- return fast;
371
- }
372
- return this.searchStored(buf, topK, among);
373
- }
374
- delete(chunkId) {
375
- this.stmt('DELETE FROM chunk_embeddings WHERE rowid = ?').run(BigInt(chunkId));
376
- this.forget(chunkId);
377
- }
378
- /** `PRAGMA data_version`. Another connection's commit changes it; this connection's own writes do not. */
379
- dataVersion() {
380
- return Number(this.stmt('PRAGMA data_version').pluck().get());
381
- }
382
- /** Dimensions the vec0 table was created with, or null when it has not been created. */
383
- tableDims() {
384
- const row = this.stmt(`SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'chunk_embeddings'`).get();
385
- const found = row?.sql.match(/FLOAT\[(\d+)\]/i);
386
- return found ? Number(found[1]) : null;
387
- }
388
- /**
389
- * The stored vectors, copied once from sqlite-vec's packed shadow blobs (one blob per chunk of vectors, not one row
390
- * per vector). The copy is dropped when the row count changes without this class writing it, or when another
391
- * connection commits (`PRAGMA data_version`): a replace there keeps the count and would otherwise stay invisible.
392
- * Past {@link MIRROR_MAX_BYTES} there is no copy. A table declared at a different dimension is not copied, so the
393
- * sqlite-vec query throws its dimension error instead of this copy answering with a slice of the blob.
394
- */
395
- mirrorReady() {
396
- let n;
397
- let version;
398
- try {
399
- n = this.stmt('SELECT COUNT(*) AS n FROM chunk_embeddings_rowids').get().n;
400
- version = this.dataVersion();
401
- }
402
- catch {
403
- this.mirror = null;
404
- return false;
405
- }
406
- if (this.mirror && this.mirror.count === n && this.mirror.version === version)
407
- return true;
408
- if (n === 0) {
409
- this.mirror = { ids: [], slot: new Map(), data: new Float32Array(0), rows: null, simdTried: false, count: 0, cap: 0, version };
410
- return true;
411
- }
412
- if (n * this.dimensions * 4 > MIRROR_MAX_BYTES) {
413
- this.mirror = null;
414
- return false;
415
- }
416
- this.mirror = this.loadMirror(n, version);
417
- return this.mirror !== null;
418
- }
419
- loadMirror(n, version) {
420
- const declared = this.tableDims();
421
- if (declared !== null && declared !== this.dimensions)
422
- return null;
423
- let chunks;
424
- let rows;
425
- try {
426
- chunks = this.stmt('SELECT rowid, vectors FROM chunk_embeddings_vector_chunks00').all();
427
- rows = this.stmt('SELECT rowid, chunk_id, chunk_offset FROM chunk_embeddings_rowids').all();
428
- }
429
- catch {
430
- return null;
431
- }
432
- // A commit landed while the blobs were being copied. This search uses the KNN path; the next one copies again.
433
- if (rows.length !== n || this.dataVersion() !== version)
434
- return null;
435
- const blobs = new Map();
436
- for (const chunk of chunks)
437
- blobs.set(Number(chunk.rowid), chunk.vectors);
438
- const dims = this.dimensions;
439
- // A reload (another connection committed) replaces this mirror, so its linear memory is taken over, not reserved again,
440
- // when the vectors it now holds still want one.
441
- const wantsRows = this.wantsRows(n);
442
- const held = this.mirror?.rows;
443
- const simd = !wantsRows ? null : held && held.capRows >= n ? held : simdScan_1.SimdRows.create(dims, n);
444
- const data = simd ? simd.data : new Float32Array(n * dims);
445
- const ids = new Array(n);
446
- const slot = new Map();
447
- try {
448
- for (let i = 0; i < n; i++) {
449
- const blob = blobs.get(Number(rows[i].chunk_id));
450
- const off = Number(rows[i].chunk_offset) * dims;
451
- if (!blob || (blob.byteOffset + off * 4) % 4 !== 0 || off * 4 + dims * 4 > blob.length)
452
- return null;
453
- data.set(new Float32Array(blob.buffer, blob.byteOffset + off * 4, dims), i * dims);
454
- const id = Number(rows[i].rowid);
455
- ids[i] = id;
456
- slot.set(id, i);
457
- }
458
- }
459
- catch {
460
- return null;
461
- }
462
- return { ids, slot, data, rows: simd, simdTried: wantsRows, count: n, cap: simd ? simd.capRows : n, version };
463
- }
464
- /**
465
- * Whether `live` vectors want a memory the SIMD scan reads. Their bytes decide, never the room the mirror keeps for more:
466
- * the capacity doubles ahead of the vectors, and a mirror that holds 4.3 MB must not get the 10 GiB reservation because
467
- * it has room for 8.6.
468
- */
469
- wantsRows(live) {
470
- return live * this.dimensions * 4 >= simdScan_1.SIMD_MIN_BYTES;
471
- }
472
- /** The live vectors have reached the minimum: move them, once, to a memory the SIMD scan reads (the JavaScript copy stays if it cannot). */
473
- moveToRows(m) {
474
- m.simdTried = true;
475
- const simd = simdScan_1.SimdRows.create(this.dimensions, m.cap);
476
- if (!simd)
477
- return;
478
- simd.data.set(m.data.subarray(0, m.count * this.dimensions));
479
- m.data = simd.data;
480
- m.rows = simd;
481
- }
482
- remember(chunkId, floats) {
483
- const m = this.mirror;
484
- if (!m)
485
- return;
486
- const dims = this.dimensions;
487
- const have = m.slot.get(chunkId);
488
- if (have !== undefined) {
489
- m.data.set(floats, have * dims);
490
- return;
491
- }
492
- if ((m.count + 1) * dims * 4 > MIRROR_MAX_BYTES) {
493
- this.mirror = null;
494
- return;
495
- }
496
- if (m.count === m.cap) {
497
- // Doubling can jump past the byte cap while the next single vector still fits (16,384 → 32,768 at 768 dims).
498
- const maxCap = Math.floor(MIRROR_MAX_BYTES / (dims * 4));
499
- let cap = Math.max(m.cap * 2, m.cap + 16, 16);
500
- if (cap > maxCap)
501
- cap = maxCap;
502
- if (cap < m.count + 1) {
503
- this.mirror = null;
504
- return;
505
- }
506
- if (m.rows) {
507
- if (!m.rows.grow(cap)) {
508
- this.mirror = null;
509
- return;
510
- }
511
- m.data = m.rows.data;
512
- }
513
- else {
514
- const data = new Float32Array(cap * dims);
515
- data.set(m.data.subarray(0, m.count * dims));
516
- m.data = data;
517
- }
518
- m.cap = cap;
519
- }
520
- const i = m.count;
521
- m.data.set(floats, i * dims);
522
- m.ids[i] = chunkId;
523
- m.slot.set(chunkId, i);
524
- m.count++;
525
- if (!m.rows && !m.simdTried && this.wantsRows(m.count))
526
- this.moveToRows(m);
527
- }
528
- forget(chunkId) {
529
- const m = this.mirror;
530
- if (!m)
531
- return;
532
- const i = m.slot.get(chunkId);
533
- if (i === undefined)
534
- return;
535
- const dims = this.dimensions;
536
- const last = m.count - 1;
537
- if (i !== last) {
538
- m.data.copyWithin(i * dims, last * dims, (last + 1) * dims);
539
- const moved = m.ids[last];
540
- m.ids[i] = moved;
541
- m.slot.set(moved, i);
542
- }
543
- m.slot.delete(chunkId);
544
- m.ids.pop();
545
- m.count--;
546
- }
547
- /** sqlite-vec's own distance for one stored vector. It is the KNN distance: the same C function, so the bits match. */
548
- l2(slot, query) {
549
- const m = this.mirror;
550
- const bytes = this.dimensions * 4;
551
- const blob = Buffer.from(m.data.buffer, m.data.byteOffset + slot * bytes, bytes);
552
- const row = this.stmt('SELECT vec_distance_l2(?, ?) AS distance').get(blob, query);
553
- return row.distance;
554
- }
555
- /** The distance from `qf` to each of the `n` vectors (the packed ones, or those at `slots`) to `approx`, by the SIMD scan where the mirror has one. */
556
- scanMirror(m, qf, slots, n, approx) {
557
- const dims = this.dimensions;
558
- if (m.rows && qf.length === dims) {
559
- try {
560
- m.rows.stage(qf);
561
- if (slots)
562
- for (let i = 0; i < n; i++)
563
- approx[i] = m.rows.distance(slots[i]);
564
- else
565
- m.rows.distances(0, n, approx, 0);
566
- return;
567
- }
568
- catch {
569
- // the scan only chooses candidates: the same search in JavaScript is the same answer
570
- }
571
- }
572
- if (slots)
573
- (0, exports.scanSlotsL2)(m.data, slots, dims, qf, approx);
574
- else
575
- (0, exports.scanPackedL2)(m.data, n, dims, qf, approx);
576
- }
577
- scanBuffer(n) {
578
- if (this.scanned.length < n)
579
- this.scanned = new Float64Array(Math.max(n, this.scanned.length * 2));
580
- return this.scanned;
581
- }
582
- rankExact(slots, query, topK) {
583
- const rows = slots.map(slot => ({ chunkId: this.mirror.ids[slot], distance: this.l2(slot, query) }));
584
- rows.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
585
- const edge = rows.length > topK ? rows[topK - 1].distance : Number.POSITIVE_INFINITY;
586
- const out = [];
587
- for (let i = 0; i < rows.length; i++) {
588
- if (i >= topK && rows[i].distance !== edge)
589
- break;
590
- out.push({ chunkId: rows[i].chunkId, score: 1 - (rows[i].distance * rows[i].distance) / 2 });
591
- }
592
- return out;
593
- }
594
- /**
595
- * Same rows as {@link searchStored}. Null means this query is cheaper, or only correct, on the KNN path: the
596
- * candidate band is most of the index, or the scan disagreed with `vec_distance_l2` by more than the slack allows.
597
- */
598
- searchMirror(qf, query, topK, among) {
599
- const m = this.mirror;
600
- let slots = null;
601
- if (among) {
602
- const seen = new Set();
603
- slots = [];
604
- for (const id of among) {
605
- if (seen.has(id))
606
- continue;
607
- seen.add(id);
608
- const slot = m.slot.get(id);
609
- if (slot !== undefined)
610
- slots.push(slot);
611
- }
612
- }
613
- const n = slots ? slots.length : m.count;
614
- if (n === 0)
615
- return [];
616
- if (n <= MIRROR_EXACT_LIMIT)
617
- return this.rankExact(slots ?? Array.from({ length: n }, (_, i) => i), query, topK);
618
- const approx = this.scanBuffer(n);
619
- this.scanMirror(m, qf, slots, n, approx);
620
- const limit = kthSmallest(approx, n, Math.min(topK, n)) + MIRROR_SLACK;
621
- const picked = [];
622
- for (let i = 0; i < n; i++)
623
- if (approx[i] <= limit)
624
- picked.push(i);
625
- if (picked.length > MIRROR_EXACT_LIMIT)
626
- return null;
627
- const chosen = picked.map(i => (slots ? slots[i] : i));
628
- const distances = chosen.map(slot => this.l2(slot, query));
629
- let worst = 0;
630
- for (let i = 0; i < picked.length; i++) {
631
- const err = Math.abs(distances[i] - approx[picked[i]]);
632
- if (err > worst)
633
- worst = err;
634
- }
635
- if (worst > MIRROR_SLACK / 2)
636
- return null;
637
- const rows = chosen.map((slot, i) => ({ chunkId: m.ids[slot], distance: distances[i] }));
638
- rows.sort((a, b) => a.distance - b.distance || a.chunkId - b.chunkId);
639
- const edge = rows.length > topK ? rows[topK - 1].distance : Number.POSITIVE_INFINITY;
640
- const out = [];
641
- for (let i = 0; i < rows.length; i++) {
642
- if (i >= topK && rows[i].distance !== edge)
643
- break;
644
- out.push({ chunkId: rows[i].chunkId, score: 1 - (rows[i].distance * rows[i].distance) / 2 });
645
- }
646
- return out;
647
- }
648
- /** The KNN query, widened until a tie at the cut is whole. The answer `search` used before the in-process scan. */
649
- searchStored(buf, topK, among) {
650
- const ids = among === undefined ? null : JSON.stringify(among);
651
- const nearest = (k) => (ids === null
652
- ? this.stmt('SELECT rowid, distance FROM chunk_embeddings WHERE embedding MATCH ? AND k = ? ORDER BY distance').all(buf, k)
653
- : this.stmt('SELECT rowid, distance FROM chunk_embeddings WHERE embedding MATCH ? AND k = ? AND rowid IN (SELECT value FROM json_each(?)) ORDER BY distance').all(buf, k, ids));
654
- let k = Math.min(topK + 1, exports.KNN_MAX_K);
655
- let rows = nearest(k);
656
- while (rows.length === k && k > topK && rows[k - 1].distance === rows[topK - 1].distance) {
657
- if (k === exports.KNN_MAX_K) {
658
- rows = (ids === null
659
- ? this.stmt('SELECT rowid, vec_distance_l2(embedding, ?) AS distance FROM chunk_embeddings').all(buf)
660
- : this.stmt('SELECT rowid, vec_distance_l2(embedding, ?) AS distance FROM chunk_embeddings WHERE rowid IN (SELECT value FROM json_each(?))').all(buf, ids))
661
- .sort((a, b) => a.distance - b.distance);
662
- break;
663
- }
664
- k = Math.min(k * 4, exports.KNN_MAX_K);
665
- rows = nearest(k);
666
- }
667
- const edge = rows.length > topK ? rows[topK - 1].distance : Number.POSITIVE_INFINITY;
668
- return rows.filter((r, i) => i < topK || r.distance === edge).map(r => ({
669
- chunkId: Number(r.rowid),
670
- score: 1 - (r.distance * r.distance) / 2
671
- }));
672
- }
673
- }
674
- exports.VectorIndex = VectorIndex;