renor-engine 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (193) hide show
  1. package/README.md +27 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +3 -3
  4. package/dist/cli.js +1656 -490
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -548
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/preferenceExtractor.js +0 -150
  75. package/dist/extraction/secretGuard.js +0 -176
  76. package/dist/extraction/secretScanner.js +0 -665
  77. package/dist/extraction/typedFacts.js +0 -224
  78. package/dist/extraction/writePathAdjudication.js +0 -110
  79. package/dist/importing/apply.js +0 -672
  80. package/dist/importing/bundle-write.js +0 -80
  81. package/dist/importing/cli.js +0 -408
  82. package/dist/importing/existing.js +0 -51
  83. package/dist/importing/facts.js +0 -121
  84. package/dist/importing/forgetImported.js +0 -79
  85. package/dist/importing/hosted.js +0 -467
  86. package/dist/importing/journalLog.js +0 -96
  87. package/dist/importing/markdown.js +0 -98
  88. package/dist/importing/plan.js +0 -201
  89. package/dist/importing/secretScreen.js +0 -113
  90. package/dist/importing/serialize.js +0 -188
  91. package/dist/importing/source.js +0 -463
  92. package/dist/importing/types.js +0 -6
  93. package/dist/indexer/chunker.js +0 -319
  94. package/dist/indexer/dailyEntries.js +0 -63
  95. package/dist/indexer/hasher.js +0 -10
  96. package/dist/indexer/laneRuns.js +0 -48
  97. package/dist/indexer/o200k.js +0 -481
  98. package/dist/indexer/outsideRoot.js +0 -77
  99. package/dist/indexer/parser.js +0 -170
  100. package/dist/indexer/pipeline.js +0 -988
  101. package/dist/indexer/staleness.js +0 -201
  102. package/dist/indexer/tokenizer.js +0 -77
  103. package/dist/indexer/walker.js +0 -170
  104. package/dist/mcp/hive.js +0 -88
  105. package/dist/mcp/requestErrors.js +0 -47
  106. package/dist/mcp/server.js +0 -426
  107. package/dist/memory/agentLease.js +0 -43
  108. package/dist/memory/confinedWrite.js +0 -315
  109. package/dist/memory/controls.js +0 -73
  110. package/dist/memory/dailyNote.js +0 -265
  111. package/dist/memory/durableAppend.js +0 -57
  112. package/dist/memory/factConflicts.js +0 -203
  113. package/dist/memory/factTerms.js +0 -321
  114. package/dist/memory/handoff.js +0 -363
  115. package/dist/memory/hiveNameHeader.cjs +0 -10
  116. package/dist/memory/inventory.js +0 -55
  117. package/dist/memory/invisibleText.js +0 -20
  118. package/dist/memory/lifecycle.js +0 -1991
  119. package/dist/memory/limits.js +0 -13
  120. package/dist/memory/requests.js +0 -183
  121. package/dist/memory/sourceName.js +0 -276
  122. package/dist/memory/statusFacts.js +0 -61
  123. package/dist/memory/whatsNew.js +0 -158
  124. package/dist/memory/writer.js +0 -175
  125. package/dist/memory/writerConflicts.js +0 -85
  126. package/dist/memorymap/memoryMap.js +0 -1594
  127. package/dist/memorymap/replacedJournal.js +0 -58
  128. package/dist/memorymap/replacements.js +0 -410
  129. package/dist/mvp/activity.js +0 -86
  130. package/dist/mvp/agentMemory.js +0 -1800
  131. package/dist/mvp/answerCheck.js +0 -211
  132. package/dist/mvp/auth.js +0 -531
  133. package/dist/mvp/cli.js +0 -75
  134. package/dist/mvp/client-store.js +0 -462
  135. package/dist/mvp/compileCache.js +0 -242
  136. package/dist/mvp/compileSeed.js +0 -33
  137. package/dist/mvp/config.js +0 -288
  138. package/dist/mvp/connectionStatus.js +0 -80
  139. package/dist/mvp/contextDistinct.js +0 -53
  140. package/dist/mvp/contextProfile.js +0 -87
  141. package/dist/mvp/contextRelevance.js +0 -413
  142. package/dist/mvp/directorySurface.js +0 -68
  143. package/dist/mvp/engine.js +0 -1078
  144. package/dist/mvp/exportAll.js +0 -374
  145. package/dist/mvp/factLanePack.js +0 -77
  146. package/dist/mvp/factMarkers.js +0 -65
  147. package/dist/mvp/factsLane.js +0 -203
  148. package/dist/mvp/hiddenSpans.js +0 -585
  149. package/dist/mvp/history.js +0 -414
  150. package/dist/mvp/hitDates.js +0 -31
  151. package/dist/mvp/hitFloor.js +0 -36
  152. package/dist/mvp/identity.js +0 -18
  153. package/dist/mvp/idle.js +0 -113
  154. package/dist/mvp/log.js +0 -118
  155. package/dist/mvp/memoryDir.js +0 -98
  156. package/dist/mvp/posthogMcp.js +0 -210
  157. package/dist/mvp/readScope.js +0 -104
  158. package/dist/mvp/readUsefulness.js +0 -222
  159. package/dist/mvp/sanitize.js +0 -48
  160. package/dist/mvp/schemaRejections.js +0 -91
  161. package/dist/mvp/scope.js +0 -127
  162. package/dist/mvp/server.js +0 -2604
  163. package/dist/mvp/serverCard.js +0 -38
  164. package/dist/mvp/storedSize.js +0 -139
  165. package/dist/mvp/telemetry.js +0 -474
  166. package/dist/net/poolFetch.js +0 -174
  167. package/dist/recall/actionRecall.js +0 -282
  168. package/dist/recall/autoRecall.js +0 -447
  169. package/dist/recall/searchRecall.js +0 -189
  170. package/dist/retrieval/adviceLines.js +0 -195
  171. package/dist/retrieval/adviceQuestion.js +0 -44
  172. package/dist/retrieval/conversationShape.js +0 -44
  173. package/dist/retrieval/dailyEntryTime.js +0 -55
  174. package/dist/retrieval/entityLookup.js +0 -162
  175. package/dist/retrieval/factsFirst.js +0 -188
  176. package/dist/retrieval/keyword.js +0 -84
  177. package/dist/retrieval/ledgerEventTime.js +0 -51
  178. package/dist/retrieval/ledgerLiveness.js +0 -83
  179. package/dist/retrieval/ledgerPresentation.js +0 -161
  180. package/dist/retrieval/noteIdentity.js +0 -57
  181. package/dist/retrieval/orchestrator.js +0 -1171
  182. package/dist/retrieval/plainWording.js +0 -236
  183. package/dist/retrieval/queryHelpers.js +0 -142
  184. package/dist/retrieval/questionSpan.js +0 -475
  185. package/dist/retrieval/readerEvents.js +0 -690
  186. package/dist/retrieval/readerLayout.js +0 -422
  187. package/dist/retrieval/relativeTimeHints.js +0 -113
  188. package/dist/retrieval/semantic.js +0 -67
  189. package/dist/router/routerCache.js +0 -106
  190. package/dist/router/sessionRouter.js +0 -726
  191. package/dist/toolRefusal.js +0 -120
  192. package/dist/watcher/watcher.js +0 -77
  193. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,481 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.countO200k = countO200k;
7
- exports.countO200kStrict = countO200kStrict;
8
- exports.o200kLoaded = o200kLoaded;
9
- const node_zlib_1 = __importDefault(require("node:zlib"));
10
- /**
11
- * An exact o200k_base token COUNTER that holds the vocabulary as three typed arrays (about 4 MiB) instead of the
12
- * 200,000-entry Map of strings that `gpt-tokenizer` builds (about 45 MiB resident once loaded, measured against a running
13
- * workspace; see docs/performance/MEMORY-PER-WORKSPACE.md). It counts; it does not decode, and it never builds the token
14
- * list.
15
- *
16
- * WHAT "EXACT" MEANS HERE. For every string it returns the length of the list `gpt-tokenizer`'s `encode` returns for it
17
- * (with no special token allowed), because it runs the same three steps: split the text with the same regular expression;
18
- * a piece that is a whole token counts one; any other piece is merged byte pair by byte pair, lowest rank first. The
19
- * vocabulary comes from `gpt-tokenizer` itself through scripts/tokenizer/gen-o200k.cjs (src/indexer/o200k.json is its
20
- * output), and tests/indexer/o200k-counter.test.ts compares this counter with `gpt-tokenizer` on a fixed corpus and on a
21
- * seeded fuzz of every text shape that has bitten a tokenizer here: multi-byte text, emoji, lone surrogates, byte-order
22
- * marks, control characters, long runs, special-token spellings.
23
- *
24
- * THE QUIRKS KEPT, because "the same count" includes them:
25
- * - `gpt-tokenizer` looks a byte key up as a STRING when the bytes are valid UTF-8, and its decoder drops a leading
26
- * byte-order mark. So a key that starts with EF BB BF and is valid is looked up WITHOUT those three bytes, and the nine
27
- * vocabulary entries that start with a mark can never be found by their own bytes. The generator packs those nine as
28
- * empty entries, which `buildTable` does not insert.
29
- * - A piece with a lone surrogate never matches a token as a whole string; it is merged over the bytes of its U+FFFD.
30
- * - `disallowedSpecial` is a choice: `countStrict` throws on the spelling of a special token the way `encode` does by
31
- * default, and `count` counts those spellings as ordinary text (`disallowedSpecial: new Set()`).
32
- *
33
- * LAZY, TWICE: nothing here runs at import (the packed vocabulary is inflated and indexed on the first count), and the
34
- * data module is a plain `require` for the same reason as tokenizer.ts: a start that never counts a token never pays for it.
35
- */
36
- /** gpt-tokenizer's O200K_TOKEN_SPLIT_REGEX, character for character. */
37
- const CONTRACTION = String.raw `'(?:[sS]|[dD]|[mM]|[tT]|[lL][lL]|[vV][eE]|[rR][eE])`;
38
- const OPTIONAL_CONTRACTION = String.raw `(?:${CONTRACTION})?`;
39
- const SPLIT_PATTERN = String.raw `[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+${OPTIONAL_CONTRACTION}|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*${OPTIONAL_CONTRACTION}|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n/]*|\s*[\r\n]+|\s+(?!\S)|\s+`;
40
- /** The eight spellings `encode` refuses by default, in the order its pattern lists them. */
41
- const SPECIAL_SPELLINGS = ['<|endoftext|>', '<|fim_prefix|>', '<|fim_middle|>', '<|fim_suffix|>', '<|im_start|>', '<|im_end|>', '<|im_sep|>', '<|endofprompt|>'];
42
- const SPECIAL_PATTERN = new RegExp(SPECIAL_SPELLINGS.map(s => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|'));
43
- let table = null;
44
- let splitter = null;
45
- let scratch = new Uint8Array(4096);
46
- const encoder = new TextEncoder();
47
- function hashBytes(b, start, end) {
48
- let h = 0x811c9dc5;
49
- for (let i = start; i < end; i++)
50
- h = Math.imul(h ^ b[i], 0x01000193);
51
- return h >>> 0;
52
- }
53
- function buildTable() {
54
- // A plain require, not an import: the data is 1.4 MB of JSON and nothing may read it before the first count. JSON, not a .ts
55
- // module, so it needs no compiler pass and a plain `require` finds it from the source tree and from dist alike
56
- // (tsc copies a JSON file named by the `import type` above).
57
- const data = require('./o200k.json');
58
- const O200K_TOKEN_COUNT = data.tokens;
59
- const packed = node_zlib_1.default.inflateSync(Buffer.from(data.deflateBase64, 'base64'));
60
- // The module cache would keep the 1.4 MB base64 string for the life of the process, and it is never read again.
61
- delete require.cache[require.resolve('./o200k.json')];
62
- const offsets = new Uint32Array(O200K_TOKEN_COUNT + 1);
63
- const bytes = new Uint8Array(packed.length);
64
- let read = 0;
65
- let write = 0;
66
- for (let rank = 0; rank < O200K_TOKEN_COUNT; rank++) {
67
- let len = 0;
68
- let shift = 0;
69
- for (;;) {
70
- const byte = packed[read++];
71
- len |= (byte & 0x7f) << shift;
72
- if ((byte & 0x80) === 0)
73
- break;
74
- shift += 7;
75
- }
76
- offsets[rank] = write;
77
- bytes.set(packed.subarray(read, read + len), write);
78
- read += len;
79
- write += len;
80
- }
81
- offsets[O200K_TOKEN_COUNT] = write;
82
- let size = 1;
83
- while (size < O200K_TOKEN_COUNT * 2)
84
- size <<= 1;
85
- const slots = new Int32Array(size);
86
- const mask = size - 1;
87
- for (let rank = 0; rank < O200K_TOKEN_COUNT; rank++) {
88
- const start = offsets[rank];
89
- const end = offsets[rank + 1];
90
- if (end === start)
91
- continue;
92
- let slot = hashBytes(bytes, start, end) & mask;
93
- while (slots[slot] !== 0)
94
- slot = (slot + 1) & mask;
95
- slots[slot] = rank + 1;
96
- }
97
- // The packed buffer is not kept: `bytes` is its own copy, trimmed to what the tokens use.
98
- return { bytes: bytes.slice(0, write), offsets, slots, mask };
99
- }
100
- /** The rank of the token spelled by key[start..end), or -1. */
101
- function rankOf(t, key, start, end) {
102
- let slot = hashBytes(key, start, end) & t.mask;
103
- const len = end - start;
104
- for (;;) {
105
- const entry = t.slots[slot];
106
- if (entry === 0)
107
- return -1;
108
- const rank = entry - 1;
109
- const from = t.offsets[rank];
110
- if (t.offsets[rank + 1] - from === len) {
111
- let i = 0;
112
- while (i < len && t.bytes[from + i] === key[start + i])
113
- i++;
114
- if (i === len)
115
- return rank;
116
- }
117
- slot = (slot + 1) & t.mask;
118
- }
119
- }
120
- /** gpt-tokenizer's own UTF-8 test (not the platform's: it differs on a few edge sequences), kept for the byte-order-mark rule. */
121
- function isValidUtf8(b, start, end) {
122
- let i = start;
123
- while (i < end) {
124
- const byte1 = b[i];
125
- let n;
126
- let cp;
127
- if (byte1 <= 0x7f) {
128
- n = 1;
129
- cp = byte1;
130
- }
131
- else if ((byte1 & 0xe0) === 0xc0) {
132
- n = 2;
133
- cp = byte1 & 0x1f;
134
- if (byte1 <= 0xc1)
135
- return false;
136
- }
137
- else if ((byte1 & 0xf0) === 0xe0) {
138
- n = 3;
139
- cp = byte1 & 0x0f;
140
- }
141
- else if ((byte1 & 0xf8) === 0xf0) {
142
- n = 4;
143
- cp = byte1 & 0x07;
144
- if (byte1 > 0xf4)
145
- return false;
146
- }
147
- else
148
- return false;
149
- if (i + n > end)
150
- return false;
151
- for (let j = 1; j < n; j++) {
152
- const byte = b[i + j];
153
- if ((byte & 0xc0) !== 0x80)
154
- return false;
155
- cp = (cp << 6) | (byte & 0x3f);
156
- }
157
- if (n === 2 && cp < 0x80)
158
- return false;
159
- if (n === 3 && cp < 2048)
160
- return false;
161
- if (n === 4 && cp < 65536)
162
- return false;
163
- if (cp >= 55296 && cp <= 57343)
164
- return false;
165
- if (cp > 1114111)
166
- return false;
167
- i += n;
168
- }
169
- return true;
170
- }
171
- /** `getBpeRankFromBytes`: a key that reads as text is looked up as the decoded string, whose leading mark the decoder drops. */
172
- function rankOfBytes(t, key, start, end) {
173
- if (end - start >= 3 && key[start] === 0xef && key[start + 1] === 0xbb && key[start + 2] === 0xbf && isValidUtf8(key, start, end)) {
174
- return rankOf(t, key, start + 3, end);
175
- }
176
- return rankOf(t, key, start, end);
177
- }
178
- /** The number of tokens `bytePairMerge` leaves for these bytes: pairs merge lowest rank first, leftmost on a tie. */
179
- function mergedCount(t, piece, length) {
180
- const starts = [];
181
- const ranks = [];
182
- const rankAt = (i, a = starts[i], b = starts[i + 2]) => {
183
- if (b === undefined)
184
- return Infinity;
185
- const r = rankOfBytes(t, piece, a, b);
186
- return r < 0 ? Infinity : r;
187
- };
188
- for (let i = 0; i <= length; i++) {
189
- starts.push(i);
190
- ranks.push(i < length - 1 ? rankAt(i, i, i + 2) : Infinity);
191
- }
192
- while (starts.length > 1) {
193
- let lowest = Infinity;
194
- let at = -1;
195
- for (let i = 0; i < ranks.length - 1; i++) {
196
- if (ranks[i] < lowest) {
197
- lowest = ranks[i];
198
- at = i;
199
- }
200
- }
201
- if (lowest === Infinity || at === -1)
202
- break;
203
- starts.splice(at + 1, 1);
204
- ranks.splice(at, 1);
205
- ranks[at] = rankAt(at);
206
- if (at > 0)
207
- ranks[at - 1] = rankAt(at - 1);
208
- }
209
- return starts.length - 1;
210
- }
211
- /** A high surrogate not followed by a low one, or a low one not preceded by a high one (String.prototype.isWellFormed, which the build's lib lacks). */
212
- const LONE_SURROGATE = /[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?<![\uD800-\uDBFF])[\uDC00-\uDFFF]/;
213
- /**
214
- * Pieces counted recently. A reply is budgeted from the same notes again and again, so the same words come back; gpt-tokenizer
215
- * kept up to 100,000 merge results for this (and ~10 MiB with them). This keeps the last few thousand and starts over when
216
- * full (~0.3 MiB at most): the count of a piece never depends on the cache, only the time.
217
- *
218
- * The table is open-addressed and looked up by the characters of the piece where they stand in the text, so a piece that is
219
- * already counted is found without making a string of it (a `Map` needs the string, and a call counts a few thousand
220
- * pieces). A string is made only for a piece that is new.
221
- */
222
- const PIECE_CACHE_LIMIT = 4096;
223
- const PIECE_SLOTS = 8192;
224
- const PIECE_MASK = PIECE_SLOTS - 1;
225
- /** '' marks a free slot: no piece is empty. Made on the first count, like the vocabulary. */
226
- let pieceKeys = null;
227
- let pieceHashes = null;
228
- let pieceCounts = null;
229
- let piecesKept = 0;
230
- /** The tokens of `text.slice(start, end)`, which is not empty. */
231
- function countPiece(t, text, start, end) {
232
- const keys = pieceKeys ??= new Array(PIECE_SLOTS).fill('');
233
- const hashes = pieceHashes ??= new Int32Array(PIECE_SLOTS);
234
- const counts = pieceCounts ??= new Int32Array(PIECE_SLOTS);
235
- const length = end - start;
236
- let hash = 0x811c9dc5;
237
- for (let k = start; k < end; k++)
238
- hash = Math.imul(hash ^ text.charCodeAt(k), 0x01000193);
239
- let slot = hash & PIECE_MASK;
240
- for (;;) {
241
- const key = keys[slot];
242
- if (key === '')
243
- break;
244
- if (hashes[slot] === hash && key.length === length) {
245
- let k = 0;
246
- while (k < length && key.charCodeAt(k) === text.charCodeAt(start + k))
247
- k++;
248
- if (k === length)
249
- return counts[slot];
250
- }
251
- slot = (slot + 1) & PIECE_MASK;
252
- }
253
- const piece = text.slice(start, end);
254
- const n = countPieceUncached(t, piece);
255
- if (piecesKept >= PIECE_CACHE_LIMIT) {
256
- keys.fill('');
257
- piecesKept = 0;
258
- slot = hash & PIECE_MASK;
259
- }
260
- keys[slot] = piece;
261
- hashes[slot] = hash;
262
- counts[slot] = n;
263
- piecesKept++;
264
- return n;
265
- }
266
- function countPieceUncached(t, piece) {
267
- // At most 3 bytes per UTF-16 unit; encodeInto never writes more.
268
- const need = piece.length * 3;
269
- if (scratch.length < need)
270
- scratch = new Uint8Array(Math.max(need, scratch.length * 2));
271
- const written = encoder.encodeInto(piece, scratch).written;
272
- // A whole-token piece counts one. Only a well-formed string can be a vocabulary key: a lone surrogate has no spelling.
273
- if (!LONE_SURROGATE.test(piece) && rankOf(t, scratch, 0, written) >= 0)
274
- return 1;
275
- return mergedCount(t, scratch, written);
276
- }
277
- /**
278
- * THE SPLIT WITHOUT THE PATTERN. SPLIT_PATTERN is the definition of where a text is cut into pieces, and it costs about 75 ns a
279
- * character (Unicode property classes, one match array and one string for every piece): the largest single part of counting a
280
- * reply, which counts a few thousand pieces. A text with no surrogate unit is cut by `pieceEnd` below instead, which makes
281
- * the same cuts by the pattern's own rules and allocates nothing.
282
- *
283
- * It is the pattern's seven alternatives, tried at each position in the pattern's order and with its backtracking spelled out
284
- * (the comments in `pieceEnd` say which). A character matters to the pattern only through which of its classes it is in,
285
- * and a unit's classes are worked out the first time it is met, by those same classes (`classify`), so the scan cannot
286
- * disagree with the pattern about a character; it can only disagree about the grammar, and tests/indexer/o200k-counter.test.ts
287
- * holds every count equal to gpt-tokenizer's over strings made of every class (all of them, up to six characters).
288
- * A text with a surrogate unit (an emoji, any character above U+FFFF, a lone half) goes through the pattern, as before.
289
- */
290
- const KNOWN = 0x80;
291
- /** [\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}] */
292
- const UPPER = 0x01;
293
- /** [\p{Ll}\p{Lm}\p{Lo}\p{M}] */
294
- const LOWER = 0x02;
295
- /** \p{L} */
296
- const LETTER = 0x04;
297
- /** \p{N} */
298
- const NUMBER = 0x08;
299
- /** \r or \n */
300
- const BREAK = 0x10;
301
- /** \s */
302
- const SPACE = 0x20;
303
- const SURROGATE_UNIT = /[\uD800-\uDFFF]/;
304
- let classes = null;
305
- let classTests = null;
306
- /** The classes of a UTF-16 unit that is not a surrogate, kept in `classes` for the next time. */
307
- function classify(unit) {
308
- const tests = classTests ??= {
309
- upper: /[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]/u, lower: /[\p{Ll}\p{Lm}\p{Lo}\p{M}]/u, letter: /\p{L}/u, number: /\p{N}/u, space: /\s/u,
310
- };
311
- const ch = String.fromCharCode(unit);
312
- let bits = KNOWN;
313
- if (tests.upper.test(ch))
314
- bits |= UPPER;
315
- if (tests.lower.test(ch))
316
- bits |= LOWER;
317
- if (tests.letter.test(ch))
318
- bits |= LETTER;
319
- if (tests.number.test(ch))
320
- bits |= NUMBER;
321
- if (unit === 0x0a || unit === 0x0d)
322
- bits |= BREAK;
323
- if (tests.space.test(ch))
324
- bits |= SPACE;
325
- classes[unit] = bits;
326
- return bits;
327
- }
328
- function classAt(cls, text, at) {
329
- const unit = text.charCodeAt(at);
330
- return cls[unit] || classify(unit);
331
- }
332
- /** The end of the optional contraction ('s 'd 'm 't 'll 've 're, either case) that may follow a word ending at `at`: `at` when there is none. */
333
- function contractionEnd(text, at, n) {
334
- if (at + 1 >= n || text.charCodeAt(at) !== 0x27)
335
- return at;
336
- // `| 0x20` reads an ASCII letter of either case as its lower case, and no other unit as one of these letters.
337
- const first = text.charCodeAt(at + 1) | 0x20;
338
- if (first === 0x73 || first === 0x64 || first === 0x6d || first === 0x74)
339
- return at + 2;
340
- if (at + 2 >= n)
341
- return at;
342
- const second = text.charCodeAt(at + 2) | 0x20;
343
- return (first === 0x6c && second === 0x6c) || ((first === 0x76 || first === 0x72) && second === 0x65) ? at + 3 : at;
344
- }
345
- /** [Upper]* [Lower]+ (contraction)? from `from`: the end, or -1. The greedy upper run gives characters back one at a time until a lower one can start. */
346
- function wordEnd(cls, text, from, n) {
347
- let upper = from;
348
- while (upper < n && (classAt(cls, text, upper) & UPPER) !== 0)
349
- upper++;
350
- for (let start = upper; start >= from; start--) {
351
- if (start < n && (classAt(cls, text, start) & LOWER) !== 0) {
352
- let end = start + 1;
353
- while (end < n && (classAt(cls, text, end) & LOWER) !== 0)
354
- end++;
355
- return contractionEnd(text, end, n);
356
- }
357
- }
358
- return -1;
359
- }
360
- /** [Upper]+ [Lower]* (contraction)? from `from`: the end, or -1. */
361
- function shoutEnd(cls, text, from, n) {
362
- let end = from;
363
- while (end < n && (classAt(cls, text, end) & UPPER) !== 0)
364
- end++;
365
- if (end === from)
366
- return -1;
367
- while (end < n && (classAt(cls, text, end) & LOWER) !== 0)
368
- end++;
369
- return contractionEnd(text, end, n);
370
- }
371
- /** Whether a character that is not a space, a letter or a number: what alternative 4 runs over. */
372
- const isOther = (bits) => (bits & (SPACE | LETTER | NUMBER)) === 0;
373
- /** The end of the piece the pattern cuts at `at` (always past it) in a text with no surrogate unit. */
374
- function pieceEnd(cls, text, at, n) {
375
- // The commonest piece of prose, [ ]?[a-z]+, found without the classes: when the unit after it is ASCII and no apostrophe,
376
- // nothing can carry the word on (of the ASCII units only a to z are lower case) and no contraction starts.
377
- let word = text.charCodeAt(at) === 0x20 ? at + 1 : at;
378
- if (word < n) {
379
- let unit = text.charCodeAt(word);
380
- if (unit >= 0x61 && unit <= 0x7a) {
381
- while (++word < n && (unit = text.charCodeAt(word)) >= 0x61 && unit <= 0x7a)
382
- ;
383
- if (word >= n || (unit < 0x80 && unit !== 0x27))
384
- return word;
385
- }
386
- }
387
- const first = classAt(cls, text, at);
388
- // 3, ahead of the words: a number is no letter and is not in front of a word, so alternatives 1 and 2 cannot match at it.
389
- // One to three numbers.
390
- if ((first & NUMBER) !== 0) {
391
- let end = at + 1;
392
- while (end < n && end < at + 3 && (classAt(cls, text, end) & NUMBER) !== 0)
393
- end++;
394
- return end;
395
- }
396
- // 1 and 2: a word, with one character in front of it that is no letter, number or line break; the word with the
397
- // character is tried first, then the word alone, in each alternative.
398
- const lead = (first & (BREAK | LETTER | NUMBER)) === 0;
399
- let end = lead ? wordEnd(cls, text, at + 1, n) : -1;
400
- if (end < 0)
401
- end = wordEnd(cls, text, at, n);
402
- if (end < 0 && lead)
403
- end = shoutEnd(cls, text, at + 1, n);
404
- if (end < 0)
405
- end = shoutEnd(cls, text, at, n);
406
- if (end >= 0)
407
- return end;
408
- // 4: [ ]?[^\s\p{L}\p{N}]+[\r\n/]*. A space in front is taken when something that is not a space, letter or number follows.
409
- if (text.charCodeAt(at) === 0x20) {
410
- if (at + 1 < n && isOther(classAt(cls, text, at + 1))) {
411
- end = at + 2;
412
- while (end < n && isOther(classAt(cls, text, end)))
413
- end++;
414
- while (end < n && (text.charCodeAt(end) === 0x0d || text.charCodeAt(end) === 0x0a || text.charCodeAt(end) === 0x2f))
415
- end++;
416
- return end;
417
- }
418
- }
419
- else if (isOther(first)) {
420
- end = at + 1;
421
- while (end < n && isOther(classAt(cls, text, end)))
422
- end++;
423
- while (end < n && (text.charCodeAt(end) === 0x0d || text.charCodeAt(end) === 0x0a || text.charCodeAt(end) === 0x2f))
424
- end++;
425
- return end;
426
- }
427
- // 5, 6 and 7: white space. 5 is \s*[\r\n]+ (the run up to its last line break), 6 is \s+(?!\S) (the run, or all of it but
428
- // the last character when a character that is not white space follows), 7 is \s+.
429
- if ((first & SPACE) !== 0) {
430
- let run = at + 1;
431
- while (run < n && (classAt(cls, text, run) & SPACE) !== 0)
432
- run++;
433
- for (let k = run - 1; k >= at; k--)
434
- if ((classAt(cls, text, k) & BREAK) !== 0)
435
- return k + 1;
436
- return run === n || run - at < 2 ? run : run - 1;
437
- }
438
- return -1;
439
- }
440
- function countBySplit(t, text) {
441
- const cls = classes ??= new Uint8Array(65536);
442
- const n = text.length;
443
- let total = 0;
444
- for (let at = 0; at < n;) {
445
- const end = pieceEnd(cls, text, at, n);
446
- if (end < 0) {
447
- at++;
448
- continue;
449
- } // no alternative matches here: the pattern moves on
450
- // A single ASCII unit is one byte, and every byte is a token.
451
- total += end - at === 1 && text.charCodeAt(at) < 0x80 ? 1 : countPiece(t, text, at, end);
452
- at = end;
453
- }
454
- return total;
455
- }
456
- function countByPattern(t, text) {
457
- const re = splitter ??= new RegExp(SPLIT_PATTERN, 'gu');
458
- let total = 0;
459
- for (const match of text.matchAll(re))
460
- total += countPiece(t, text, match.index, match.index + match[0].length);
461
- return total;
462
- }
463
- function countText(text) {
464
- const t = table ??= buildTable();
465
- return SURROGATE_UNIT.test(text) ? countByPattern(t, text) : countBySplit(t, text);
466
- }
467
- /** Tokens in `text`, special-token spellings counted as ordinary text (gpt-tokenizer's `{ disallowedSpecial: new Set() }`). */
468
- function countO200k(text) {
469
- return countText(text);
470
- }
471
- /** Tokens in `text`, throwing on a special-token spelling the way gpt-tokenizer's default `encode` does. */
472
- function countO200kStrict(text) {
473
- const match = text.match(SPECIAL_PATTERN);
474
- if (match !== null)
475
- throw new Error(`Disallowed special token found: ${match[0]}`);
476
- return countText(text);
477
- }
478
- /** True once the vocabulary has been inflated. Diagnostics and tests. */
479
- function o200kLoaded() {
480
- return table !== null;
481
- }
@@ -1,77 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.removeRowsOutsideRoot = removeRowsOutsideRoot;
7
- exports.reindexPending = reindexPending;
8
- exports.clearReindexPending = clearReindexPending;
9
- const path_1 = __importDefault(require("path"));
10
- const sqliteVec = require('sqlite-vec');
11
- /**
12
- * Remove the index's rows for files outside the folder it is opened on.
13
- *
14
- * The index names each note by its absolute path. A workspace folder moved or restored from a backup into another
15
- * directory therefore leaves every row of the old location behind, and a search serves each note twice: once from
16
- * the new folder, and once from the old path, which no longer exists. This removes the rows whose path is absolute and
17
- * not under `memoryDir`: the note, and what the note made (its chunks, entities, structured rows and lane runs go with
18
- * it by cascade), plus the facts, relations and write events that name it, and the vectors of its chunks.
19
- *
20
- * Rows that name no file are kept: the engine's lifecycle facts (source `_mcp_`), the ledger's events and the agents'
21
- * requests and cursors are not made from a note's path, so the folder's move does not touch them. Rows for notes under
22
- * the folder are kept as they are, so an unmoved folder removes nothing and its index is not changed.
23
- *
24
- * The index is derived from the Markdown, so what is removed is indexed again by the next pass over the folder, at the
25
- * paths it has now (`engineIndex`). The removal and the mark that the folder is owed a pass (`reindexPending`) are one
26
- * transaction: a process that ends after it finds the mark on its next start, however long the pass would have taken.
27
- * The canonical notes and the ledger are never written.
28
- */
29
- function removeRowsOutsideRoot(db, memoryDir) {
30
- const root = path_1.default.resolve(memoryDir);
31
- const params = { root, under: root.endsWith(path_1.default.sep) ? root : root + path_1.default.sep };
32
- // Absolute, and neither the folder itself nor below it. `length(@under)` counts characters as `substr` does.
33
- const outside = (column) => `(substr(${column}, 1, 1) = '/' AND ${column} <> @root AND substr(${column}, 1, length(@under)) <> @under)`;
34
- return db.transaction(() => {
35
- const removed = { notes: 0, facts: 0, relations: 0, events: 0, vectors: 0 };
36
- // Facts first: fact_terms is keyed by fact id and has no cascade, so its rows go with the facts that name the note.
37
- const factIds = `SELECT id FROM facts WHERE ${outside('source_path')}`;
38
- db.prepare(`DELETE FROM fact_terms WHERE fact_id IN (${factIds})`).run(params);
39
- removed.facts = db.prepare(`DELETE FROM facts WHERE ${outside('source_path')}`).run(params).changes;
40
- removed.relations = db.prepare(`DELETE FROM relations WHERE ${outside('source_note')}`).run(params).changes;
41
- removed.events = db.prepare(`DELETE FROM write_events WHERE ${outside('note_path')}`).run(params).changes;
42
- // Notes last: chunks, entities, structured rows and lane runs cascade from them, and chunks_fts and the note
43
- // identity index follow their triggers.
44
- removed.notes = db.prepare(`DELETE FROM notes WHERE ${outside('path')}`).run(params).changes;
45
- // Vectors are keyed by chunk id with no cascade: those of the chunks just removed are orphans now. Only a removed
46
- // note can leave one, so an index with nothing to remove is not read for them.
47
- const vectorTable = db.prepare("SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'chunk_embeddings'").get();
48
- if (removed.notes > 0 && vectorTable !== undefined) {
49
- // The table is a vec0 virtual table: this connection can only read it once the extension is loaded on it.
50
- sqliteVec.load(db);
51
- const orphans = db.prepare('SELECT rowid FROM chunk_embeddings WHERE rowid NOT IN (SELECT id FROM chunks)').pluck().all();
52
- if (orphans.length > 0) {
53
- db.prepare('DELETE FROM chunk_embeddings WHERE rowid IN (SELECT value FROM json_each(?))').run(JSON.stringify(orphans));
54
- }
55
- removed.vectors = orphans.length;
56
- }
57
- const total = removed.notes + removed.facts + removed.relations + removed.events + removed.vectors;
58
- if (total > 0) {
59
- db.prepare(`INSERT INTO index_state (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value`)
60
- .run(REINDEX_PENDING, params.root);
61
- }
62
- return { ...removed, total };
63
- })();
64
- }
65
- const REINDEX_PENDING = 'reindex_pending';
66
- /**
67
- * True while the folder is owed an index pass: a removal made it, and no pass has since finished without a failure.
68
- * A start reads this rather than the count of rows it removed, so a start that is cut short does not leave an index
69
- * that nothing indexes again.
70
- */
71
- function reindexPending(db) {
72
- return db.prepare('SELECT 1 FROM index_state WHERE key = ?').get(REINDEX_PENDING) !== undefined;
73
- }
74
- /** Clears the mark once the folder has been indexed at its current paths with no failure. */
75
- function clearReindexPending(db) {
76
- db.prepare('DELETE FROM index_state WHERE key = ?').run(REINDEX_PENDING);
77
- }