renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,205 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.MAX_DRIFT_VALUES = void 0;
4
- exports.nameOf = nameOf;
5
- exports.related = related;
6
- exports.possibleDuplicates = possibleDuplicates;
7
- exports.namedConflicts = namedConflicts;
8
- const rowStore_1 = require("../context/rowStore");
9
- const queryHelpers_1 = require("../retrieval/queryHelpers");
10
- const factTerms_1 = require("./factTerms");
11
- const scope_1 = require("../mvp/scope");
12
- const sourceName_1 = require("./sourceName");
13
- /**
14
- * Multi-agent drift: current lifecycle facts that may be one thing, named, never merged.
15
- *
16
- * A correction closes only the current fact of the same subject (one identity: case and Unicode form folded),
17
- * predicate and scope (any spelling of it, scopeSpellings). When several agents write, the same thing gets saved under another name (Codex writes
18
- * "Atlas project", Claude Code writes "Atlas") or in another scope (a correction saved without the project's scope
19
- * goes to global), and then both values stay current. This module supersedes nothing across subjects or scopes.
20
- * It names what may be one thing, so the agent or the user can decide:
21
- *
22
- * - `possibleDuplicates`, on a fact save's receipt: the other current facts related to the one just written;
23
- * - `conflicts`, on a read: groups of related current facts whose values disagree, for the facts the read returns.
24
- *
25
- * Two facts are related when they have the same predicate and their subjects are one identity (`identityKey`), or
26
- * one name is the other with one more word before or after it: "Atlas" and "Atlas project", "staging database" and
27
- * "Atlas staging database". A name longer by more words ("Atlas service shipments") or a possessive ("Ravi's
28
- * manager", "user's mother") names another thing. Values are compared after `normDedupeValue`, so "Sao Paulo" and
29
- * "São Paulo" do not disagree.
30
- *
31
- * Scope stays an isolation boundary. A call made in a scope sees that scope's facts only, as its reads do
32
- * (`scopeFilterFor`), so it is never told what another scope holds. A call without a scope sees every scope, so it
33
- * is the one told that a subject and predicate have another current value elsewhere.
34
- *
35
- * Cost: one pass over the current lifecycle facts that reads the subject names of those with an asked predicate, then
36
- * the facts whose name passes a word test in full. Measured with 12 asked facts, 6 of them sharing their predicate with
37
- * a quarter of the facts (host load about 45): 0.4 ms at 150 current facts, 2.4 ms at 2,000, 17 ms at 20,000. Nothing
38
- * is written and nothing is embedded.
39
- */
40
- /** The most values a conflict, or a receipt's duplicates, lists: the newest first, with the total when there are more. */
41
- exports.MAX_DRIFT_VALUES = 8;
42
- /** A folded word that ends in an apostrophe and s ("ravi's", "ravi’s") or in s and an apostrophe ("users'"). */
43
- const POSSESSIVE = /[\p{L}\p{N}](?:['’]s|s['’])(?=\s|$)/u;
44
- /** The fold identityKey applies before its alias table: NFC, trimmed, lower-cased. */
45
- function fold(name) {
46
- return name.normalize('NFC').trim().toLowerCase().normalize('NFC');
47
- }
48
- /** What separates two words of a folded name. A word keeps its hyphens and underscores: "atl-412" is one word. */
49
- const NOT_A_WORD = /[^\p{L}\p{N}\p{M}_-]+/u;
50
- /** The words of a subject name, by name: a read reads the name of every current fact with an asked predicate, and the names are the same read after read. */
51
- const wordsOfName = new Map();
52
- const MAX_WORDS_OF_NAME = 8192;
53
- function wordsOf(subject) {
54
- let words = wordsOfName.get(subject);
55
- if (words === undefined) {
56
- const folded = fold(subject);
57
- const split = POSSESSIVE.test(folded) ? [] : folded.split(NOT_A_WORD).filter(Boolean);
58
- words = split.length > 0 ? split : null;
59
- if (wordsOfName.size >= MAX_WORDS_OF_NAME)
60
- wordsOfName.clear();
61
- wordsOfName.set(subject, words);
62
- }
63
- return words;
64
- }
65
- function nameOf(subject, names) {
66
- let name = names.get(subject);
67
- if (!name) {
68
- name = { subject, words: wordsOf(subject) };
69
- names.set(subject, name);
70
- }
71
- return name;
72
- }
73
- /** Whether `longer` is `shorter` plus one word at its start or its end. */
74
- function oneWordLonger(longer, shorter) {
75
- if (longer.length !== shorter.length + 1)
76
- return false;
77
- const same = (offset) => shorter.every((word, i) => longer[i + offset] === word);
78
- return same(0) || same(1);
79
- }
80
- function related(a, b) {
81
- if ((a.key ??= (0, queryHelpers_1.identityKey)(a.subject)) === (b.key ??= (0, queryHelpers_1.identityKey)(b.subject)))
82
- return true;
83
- if (!a.words || !b.words)
84
- return false;
85
- return oneWordLonger(a.words, b.words) || oneWordLonger(b.words, a.words);
86
- }
87
- /**
88
- * The current lifecycle facts, in one scope or (null) every scope, newest first, that have the predicate of an asked
89
- * fact and a subject that may be related to its subject; `related` then decides. A related name has the asked name's
90
- * first word first, or its last word last: it is the same words, or one more, or one fewer at its start or end. So
91
- * one pass reads the subject name of every current fact with an asked predicate, and only the facts whose name passes
92
- * that test are read in full. A possessive asked name is related by identity alone, so it is looked for whole; an
93
- * alias is looked for by each of its spellings. The partial index of current lifecycle facts holds the predicate, so
94
- * superseded facts are never visited: `INDEXED BY` states it, because the planner would otherwise take the
95
- * source_path index, which holds every lifecycle fact ever saved.
96
- */
97
- function candidates(db, asked, scope, names) {
98
- const firsts = new Set();
99
- const lasts = new Set();
100
- const wholes = new Set();
101
- for (const { subject } of asked) {
102
- for (const spelling of (0, queryHelpers_1.identityNames)(subject)) {
103
- const { words } = nameOf(spelling, names);
104
- if (words) {
105
- firsts.add(words[0]);
106
- lasts.add(words[words.length - 1]);
107
- }
108
- else {
109
- wholes.add(fold(spelling));
110
- }
111
- }
112
- }
113
- const inScope = scope === null ? null : (0, scope_1.savedScopeSqlPredicate)((0, scope_1.normalizeScope)(scope), 'f.scope'); // any spelling of it
114
- const named = (0, factTerms_1.prepareOnce)(db, `SELECT f.id, e.canonical_name
115
- FROM facts f INDEXED BY idx_facts_current
116
- JOIN entities e ON e.id = f.subject_entity_id
117
- WHERE f.valid_to IS NULL AND f.source_path = '_mcp_'
118
- AND f.predicate IN (SELECT value FROM json_each(?))${inScope ? `
119
- AND ${inScope.sql}` : ''}`).raw().all(JSON.stringify([...new Set(asked.map(fact => fact.predicate))]), ...(inScope?.params ?? []));
120
- const ids = named.filter(([, subject]) => {
121
- const { words } = nameOf(subject, names);
122
- return words ? firsts.has(words[0]) || lasts.has(words[words.length - 1]) : wholes.has(fold(subject));
123
- }).map(([id]) => id);
124
- if (ids.length === 0)
125
- return [];
126
- return (0, factTerms_1.prepareOnce)(db, `SELECT f.id AS id, e.canonical_name AS subject, f.predicate AS predicate, COALESCE(f.object_text, '') AS object,
127
- f.scope AS scope, f.valid_from AS validFrom, f.author AS author
128
- FROM json_each(?) c
129
- CROSS JOIN facts f ON f.id = c.value
130
- JOIN entities e ON e.id = f.subject_entity_id
131
- ORDER BY f.valid_from DESC, f.id DESC`).all(JSON.stringify(ids));
132
- }
133
- /** A value as a read shows it: who saved it, when it is another agent's or an import's (memory/sourceName.ts). */
134
- function shown(row, reader) {
135
- return { subject: row.subject, object: row.object, scope: row.scope, validFrom: row.validFrom, ...(0, sourceName_1.authorMarker)(row.author, reader) };
136
- }
137
- /**
138
- * The receipt of a fact save: the other current facts with its predicate whose subject is related to the one written,
139
- * newest first. `factId` is the fact the save left current (it is not its own duplicate); `scope` is the save's scope
140
- * filter (null when the call named no scope).
141
- */
142
- function possibleDuplicates(db, written, factId, scope, reader) {
143
- const names = new Map();
144
- const asked = nameOf(written.subject, names);
145
- const found = candidates(db, [written], scope, names)
146
- .filter(row => row.id !== factId && related(asked, nameOf(row.subject, names)));
147
- return { values: found.slice(0, exports.MAX_DRIFT_VALUES).map(row => shown(row, reader)), total: found.length };
148
- }
149
- /**
150
- * The conflicts behind the current lifecycle facts a read returns: for each of them, the current facts related to it
151
- * (its own row included); every group that holds two different values is one conflict. Groups that share a fact are
152
- * one conflict. Conflicts come in the order of the first fact that names them. `scope` is the read's scope filter
153
- * (null when the read named no scope).
154
- */
155
- function namedConflicts(db, facts, scope, reader) {
156
- if (facts.length === 0)
157
- return [];
158
- const names = new Map();
159
- const rows = candidates(db, facts, scope, names);
160
- if (rows.length < 2)
161
- return [];
162
- const parent = rows.map((_, i) => i);
163
- const root = (i) => {
164
- while (parent[i] !== i)
165
- i = parent[i] = parent[parent[i]];
166
- return i;
167
- };
168
- const matched = new Set(); // rows related to a fact the read returned; only these are ever joined
169
- const firstOfEach = []; // one row of each returned fact's group, in the order the read returned the facts
170
- for (const fact of facts) {
171
- const asked = nameOf(fact.subject, names);
172
- let first = -1;
173
- rows.forEach((row, i) => {
174
- if (row.predicate !== fact.predicate || !related(asked, nameOf(row.subject, names)))
175
- return;
176
- matched.add(i);
177
- if (first < 0)
178
- first = i;
179
- else
180
- parent[root(i)] = root(first);
181
- });
182
- if (first >= 0)
183
- firstOfEach.push(first);
184
- }
185
- const groups = new Map();
186
- for (const i of [...matched].sort((a, b) => a - b)) { // row order: newest first
187
- const group = groups.get(root(i));
188
- if (group)
189
- group.push(rows[i]);
190
- else
191
- groups.set(root(i), [rows[i]]);
192
- }
193
- const conflicts = [];
194
- for (const r of new Set(firstOfEach.map(root))) {
195
- const group = groups.get(r);
196
- if (new Set(group.map(row => (0, rowStore_1.normDedupeValue)(row.object))).size < 2)
197
- continue;
198
- conflicts.push({
199
- predicate: group[0].predicate,
200
- values: group.slice(0, exports.MAX_DRIFT_VALUES).map(row => shown(row, reader)),
201
- ...(group.length > exports.MAX_DRIFT_VALUES ? { total: group.length } : {}),
202
- });
203
- }
204
- return conflicts;
205
- }
@@ -1,321 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.MAX_COMMON_WORDS = exports.RECENT_WINDOW = exports.COMPLETE_POSTINGS = exports.prepareOnce = void 0;
4
- exports.words = words;
5
- exports.atoms = atoms;
6
- exports.atomSequence = atomSequence;
7
- exports.canonicalSubject = canonicalSubject;
8
- exports.factVocabulary = factVocabulary;
9
- exports.indexFactTerms = indexFactTerms;
10
- exports.unindexFactTerms = unindexFactTerms;
11
- exports.ensureFactTerms = ensureFactTerms;
12
- exports.statementTerms = statementTerms;
13
- exports.findCurrentFacts = findCurrentFacts;
14
- exports.newestFactEvent = newestFactEvent;
15
- exports.currentFactById = currentFactById;
16
- exports.recentCurrentFacts = recentCurrentFacts;
17
- const statements_1 = require("../db/statements");
18
- Object.defineProperty(exports, "prepareOnce", { enumerable: true, get: function () { return statements_1.prepareOnce; } });
19
- const scope_1 = require("../mvp/scope");
20
- /**
21
- * The words a lifecycle fact is found by, and the index that finds facts by them.
22
- *
23
- * `fact_terms` is an inverted index over the facts table: one row per (word, scope, fact) for every
24
- * lifecycle fact, current or superseded, written by the transaction that writes the fact. A search
25
- * reads the rows of the query's words and joins only the facts they name, so a fact is found by its
26
- * words however old it is, and the work a read does is bounded (below), not a function of the table.
27
- *
28
- * How far a read goes down a word's rows. A word is *selective* when at most COMPLETE_POSTINGS rows
29
- * of the scope read have it (the rows of superseded facts count): its rows are read completely, so
30
- * every current fact that has it is found, however old, and is counted exactly, however many other
31
- * selective words the query has. A word with more rows is *common*: its rows are not read, and the
32
- * facts it finds are those among the RECENT_WINDOW newest current facts of the scope. So a table of
33
- * at most COMPLETE_POSTINGS facts is always read exactly, and so is any query with a selective word.
34
- * What is given up, and all of it: a fact that has none of the query's selective words and is older
35
- * than the newest RECENT_WINDOW facts is not found through common words alone, and an any-words read
36
- * ranks by at most MAX_COMMON_WORDS common words. Every candidate is ranked by the number of the
37
- * query's words it has.
38
- *
39
- * The work is bounded by the query, not by the table: a word costs at most COMPLETE_POSTINGS + 1
40
- * rows to classify, and a read collects at most COMPLETE_POSTINGS facts per selective word, so at
41
- * most (words in the query) x COMPLETE_POSTINGS candidates, plus RECENT_WINDOW; the tools cap a query
42
- * at 1,000 characters. Nothing is dropped to keep it so.
43
- *
44
- * It is derived data, like the rest of index.db: `ensureFactTerms` indexes whatever is missing (a
45
- * database from before the index, or one a release without it has written to). Rows are never
46
- * removed: a superseded fact keeps its rows, and the join to `facts` filters it out as current.
47
- *
48
- * No code deletes a lifecycle fact (a correction closes it). Code that ever does must delete its
49
- * rows here too, or the words of an erased fact stay in index.db. The table has no foreign key to
50
- * `facts` on purpose: deleting a note's facts would then scan this whole table once per fact.
51
- */
52
- /** The words of `value` in the order they occur, repeats included. */
53
- function tokens(value) {
54
- return value
55
- .toLowerCase()
56
- .replace(/[^\p{L}\p{N}\p{M}_-]+/gu, ' ')
57
- .split(/\s+/)
58
- .map(w => w.trim())
59
- .filter(w => w.length >= 2);
60
- }
61
- function words(value) {
62
- return Array.from(new Set(tokens(value)));
63
- }
64
- /**
65
- * Like `words`, but slugs and snake_case split into their parts, the way the full-text index
66
- * reads them: `renor-memory-1002` is renor, memory and 1002.
67
- */
68
- function atoms(value) {
69
- return Array.from(new Set(atomSequence(value)));
70
- }
71
- /** `atoms` in the order they occur, repeats included: which words are next to each other. */
72
- function atomSequence(value) {
73
- return tokens(value.replace(/[-_]+/g, ' '));
74
- }
75
- function predicateWords(predicate) {
76
- const out = new Set();
77
- const irregularBases = {
78
- does: 'do',
79
- has: 'have',
80
- };
81
- for (const word of atoms(predicate)) {
82
- out.add(word);
83
- // Relation keys commonly use third-person singular verbs while users ask
84
- // with the base form. Keep this intentionally bounded so predicate nouns
85
- // such as "status" do not get arbitrary stems.
86
- const irregular = irregularBases[word];
87
- if (irregular) {
88
- out.add(irregular);
89
- }
90
- else if (/^[a-z]+ies$/.test(word) && word.length >= 5) {
91
- out.add(`${word.slice(0, -3)}y`);
92
- }
93
- else if (/^[a-z]+(ches|shes|xes|zes|oes|sses)$/.test(word)) {
94
- out.add(word.slice(0, -2));
95
- }
96
- else if (/^[a-z]+s$/.test(word)
97
- && word.length >= 4
98
- && !word.endsWith('ss')
99
- && !word.endsWith('us')
100
- && !word.endsWith('is')) {
101
- out.add(word.slice(0, -1));
102
- }
103
- }
104
- return Array.from(out);
105
- }
106
- /** The name a read shows for a subject: the entity's, which may be an earlier spelling of an event's. */
107
- function canonicalSubject(db, entityId) {
108
- return (0, statements_1.prepareOnce)(db, 'SELECT canonical_name FROM entities WHERE id = ?').get(entityId).canonical_name;
109
- }
110
- /** Every word a fact is found by: its subject and object, and its predicate with common verb forms. */
111
- function factVocabulary(fact) {
112
- return new Set([...atoms(`${fact.subject} ${fact.object}`), ...predicateWords(fact.predicate)]);
113
- }
114
- /**
115
- * Indexes one fact. The empty term is a marker, not a word (a query word is never empty): it says
116
- * the fact has been indexed, so one whose text has no words is not indexed again and again.
117
- */
118
- function indexFactTerms(db, factId, scope, fact) {
119
- (0, statements_1.prepareOnce)(db, `INSERT OR IGNORE INTO fact_terms (term, scope, fact_id) SELECT value, ?, ? FROM json_each(?)`)
120
- .run(scope, factId, JSON.stringify(['', ...factVocabulary(fact)]));
121
- }
122
- /** Removes what indexFactTerms wrote for one fact (a retired fact: memory/lifecycle.ts). */
123
- function unindexFactTerms(db, factId, scope, fact) {
124
- (0, statements_1.prepareOnce)(db, `DELETE FROM fact_terms WHERE scope = ? AND fact_id = ? AND term IN (SELECT value FROM json_each(?))`)
125
- .run(scope, factId, JSON.stringify(['', ...factVocabulary(fact)]));
126
- }
127
- /** Indexes every lifecycle fact that is not indexed yet, in one transaction; returns how many. */
128
- function ensureFactTerms(db) {
129
- const unindexed = db.prepare(`SELECT f.id AS id, f.scope AS scope, e.canonical_name AS subject, f.predicate AS predicate, f.object_text AS object
130
- FROM facts f JOIN entities e ON e.id = f.subject_entity_id
131
- WHERE f.source_path = '_mcp_'
132
- AND NOT EXISTS (SELECT 1 FROM fact_terms t WHERE t.term = '' AND t.scope = f.scope AND t.fact_id = f.id)`).all();
133
- if (unindexed.length === 0)
134
- return 0;
135
- db.transaction(() => {
136
- for (const fact of unindexed)
137
- indexFactTerms(db, fact.id, fact.scope, { ...fact, object: fact.object ?? '' });
138
- })();
139
- return unindexed.length;
140
- }
141
- /**
142
- * The words of a fact's saved statement, kept with the fact (`facts.statement_words`) so the facts lane can count them
143
- * toward a question's words (agentMemory.ts `rankCandidateFacts`). The atoms in the order they occur, space-joined: an
144
- * atom never holds a space. Null when the fact was saved with no statement (save_memory) or has no word in it.
145
- */
146
- function statementTerms(statement) {
147
- if (statement === undefined)
148
- return null;
149
- const terms = atoms(statement);
150
- return terms.length > 0 ? terms.join(' ') : null;
151
- }
152
- const FACT_COLUMNS = `f.event_id AS eventId, e.canonical_name AS subject, f.predicate AS predicate,
153
- f.object_text AS object, f.valid_from AS validFrom, f.scope AS scope, f.author AS author,
154
- f.statement_words AS statementWords`;
155
- const FACT_MATCH_COLUMNS = `f.id AS id, ${FACT_COLUMNS}`;
156
- /** A word with at most this many rows (in the scope read) is selective: its rows are read completely. */
157
- exports.COMPLETE_POSTINGS = 1_000;
158
- /** A common word is matched among this many of the newest current facts of the scope. */
159
- exports.RECENT_WINDOW = 500;
160
- /** An any-words read ranks by at most this many common words (the first in the query); selective words all count. */
161
- exports.MAX_COMMON_WORDS = 8;
162
- /**
163
- * The query's words that some fact has, selective ones first and the rarest of them first. Reads at
164
- * most COMPLETE_POSTINGS + 1 rows of a word, which is enough to know it is common. With `needAll`,
165
- * a word no fact has ends the read: nothing can have every word.
166
- */
167
- function readWords(db, terms, scope, needAll) {
168
- const inScope = scope === undefined ? undefined : (0, scope_1.savedScopeSqlPredicate)((0, scope_1.normalizeScope)(scope), 'scope');
169
- const read = (0, statements_1.prepareOnce)(db, `SELECT fact_id FROM fact_terms WHERE term = ? ${inScope === undefined ? '' : `AND ${inScope.sql}`} LIMIT ?`).pluck();
170
- const found = [];
171
- for (const term of terms) {
172
- const ids = read.all(term, ...(inScope?.params ?? []), exports.COMPLETE_POSTINGS + 1);
173
- if (ids.length === 0) {
174
- if (needAll)
175
- return null;
176
- continue;
177
- }
178
- const common = ids.length > exports.COMPLETE_POSTINGS;
179
- found.push({ term, ids: common ? [] : ids, common });
180
- }
181
- return found.sort((a, b) => Number(a.common) - Number(b.common) || a.ids.length - b.ids.length);
182
- }
183
- /**
184
- * The rows `read` returns for each spelling of `scope` (scopeSpellings), merged newest first and cut to `limit`.
185
- * `read` takes a spelling and a limit and returns at most that many rows of a partial index, newest first, so the
186
- * merge never sorts more than one `limit` per spelling, and a scope with one spelling (global) is returned as read.
187
- */
188
- function newestOfSpellings(read, scope, limit) {
189
- const spellings = (0, scope_1.scopeSpellings)((0, scope_1.normalizeScope)(scope));
190
- const rows = spellings.flatMap(spelling => read.all(spelling, limit));
191
- if (spellings.length === 1)
192
- return rows;
193
- return rows
194
- .sort((a, b) => (a.validFrom === b.validFrom ? b.id - a.id : a.validFrom < b.validFrom ? 1 : -1))
195
- .slice(0, limit);
196
- }
197
- /** The ids of the newest current facts of the scope: `RECENT_WINDOW` rows of a partial index per spelling, in order. */
198
- function recentFactIds(db, scope) {
199
- const read = (0, statements_1.prepareOnce)(db, `SELECT f.id AS id, f.valid_from AS validFrom FROM facts f INDEXED BY ${scope === undefined ? 'idx_facts_lifecycle_recent' : 'idx_facts_lifecycle_scope_recent'}
200
- WHERE f.valid_to IS NULL AND f.source_path = '_mcp_' ${scope === undefined ? '' : 'AND f.scope = ?'}
201
- ORDER BY f.valid_from DESC, f.id DESC LIMIT ?`);
202
- const rows = scope === undefined
203
- ? read.all(exports.RECENT_WINDOW)
204
- : newestOfSpellings(read, scope, exports.RECENT_WINDOW);
205
- return rows.map(row => row.id);
206
- }
207
- /**
208
- * The current facts found by `terms`, best first: more of the words, then newest. `scope` is one
209
- * scope when given (a normalized scope, read under every spelling of it: scopeSpellings) and every
210
- * scope when omitted. Only current lifecycle facts count.
211
- *
212
- * `any` ranks the facts that have at least one word; `all` needs every word. How much of a word's
213
- * rows is read, and what that gives up, is in the header of this file.
214
- */
215
- function findCurrentFacts(db, args) {
216
- const { scope, match, limit } = args;
217
- const terms = [...new Set(args.terms)].filter(term => term !== ''); // '' is the marker row, never a word
218
- if (terms.length === 0 || limit < 1)
219
- return [];
220
- const words = readWords(db, terms, scope, match === 'all');
221
- if (!words || words.length === 0)
222
- return [];
223
- const scoped = scope === undefined ? undefined : (0, scope_1.savedScopeSqlPredicate)((0, scope_1.normalizeScope)(scope), 'f.scope');
224
- const inScope = scoped === undefined ? '' : `AND ${scoped.sql}`;
225
- const scopeArgs = scoped?.params ?? [];
226
- if (match === 'any') {
227
- // Every selective word is read completely; only the common words are matched through the newest
228
- // facts, and at most MAX_COMMON_WORDS of them are probed.
229
- const probed = words.filter(word => word.common).slice(0, exports.MAX_COMMON_WORDS);
230
- // Fact id -> how many selective words it has, counted from the rows already read.
231
- const candidates = new Map();
232
- for (const word of words)
233
- if (!word.common)
234
- for (const id of word.ids)
235
- candidates.set(id, (candidates.get(id) ?? 0) + 1);
236
- if (probed.length > 0)
237
- for (const id of recentFactIds(db, scope))
238
- if (!candidates.has(id))
239
- candidates.set(id, 0);
240
- // The candidates are ranked on their id, date and count alone; only the `limit` best are joined to
241
- // their entity and read in full. A window fact that has none of the probed words scores 0 and is dropped.
242
- return (0, statements_1.prepareOnce)(db, `WITH scored AS MATERIALIZED (
243
- SELECT f.id AS id, f.valid_from AS validFrom,
244
- json_extract(c.value, '$[1]')${probed.length === 0 ? '' : `
245
- + (SELECT COUNT(*) FROM fact_terms p
246
- WHERE p.term IN (${probed.map(() => '?').join(', ')}) AND p.scope = f.scope AND p.fact_id = f.id)`} AS shared
247
- FROM json_each(?) c
248
- CROSS JOIN facts f ON f.id = json_extract(c.value, '$[0]')
249
- WHERE f.valid_to IS NULL AND f.source_path = '_mcp_' ${inScope}
250
- ), best AS (
251
- SELECT id, validFrom, shared FROM scored WHERE shared > 0
252
- ORDER BY shared DESC, validFrom DESC, id DESC
253
- LIMIT ?
254
- )
255
- SELECT ${FACT_MATCH_COLUMNS}, b.shared AS shared
256
- FROM best b
257
- JOIN facts f ON f.id = b.id
258
- JOIN entities e ON e.id = f.subject_entity_id
259
- ORDER BY b.shared DESC, b.validFrom DESC, b.id DESC`).all(...probed.map(word => word.term), JSON.stringify([...candidates]), ...scopeArgs, limit);
260
- }
261
- // `all`: every fact that has every word has the rarest selective word, so its rows are the
262
- // candidates and the other words are probed, one candidate at a time. With no selective word, the
263
- // candidates are the newest facts and every word is probed.
264
- const driver = words.find(word => !word.common);
265
- const candidates = driver ? driver.ids : recentFactIds(db, scope);
266
- const probed = words.filter(word => word !== driver).map(word => word.term);
267
- return (0, statements_1.prepareOnce)(db, `SELECT ${FACT_MATCH_COLUMNS}, ${terms.length} AS shared
268
- FROM json_each(?) c
269
- CROSS JOIN facts f ON f.id = c.value
270
- JOIN entities e ON e.id = f.subject_entity_id
271
- WHERE f.valid_to IS NULL AND f.source_path = '_mcp_' ${inScope}
272
- ${probed.map(() => 'AND EXISTS (SELECT 1 FROM fact_terms p WHERE p.term = ? AND p.scope = f.scope AND p.fact_id = f.id)').join('\n ')}
273
- ORDER BY f.valid_from DESC, f.id DESC
274
- LIMIT ?`).all(JSON.stringify(candidates), ...scopeArgs, ...probed, limit);
275
- }
276
- /**
277
- * The ledger event that wrote the newest lifecycle fact (current or superseded) of one scope, or of
278
- * every scope when none is given. A scope's newest fact is the last row of its marker in the term
279
- * index, one seek per spelling of the scope (scopeSpellings); without a scope it is the last row of
280
- * the facts table.
281
- */
282
- function newestFactEvent(db, scope) {
283
- if (scope === undefined) {
284
- const row = (0, statements_1.prepareOnce)(db, `SELECT event_id AS eventId FROM facts WHERE source_path = '_mcp_' AND event_id IS NOT NULL ORDER BY id DESC LIMIT 1`).get();
285
- return row?.eventId;
286
- }
287
- const read = (0, statements_1.prepareOnce)(db, `SELECT t.fact_id AS id, f.event_id AS eventId
288
- FROM fact_terms t CROSS JOIN facts f ON f.id = t.fact_id
289
- WHERE t.term = '' AND t.scope = ? AND f.source_path = '_mcp_' AND f.event_id IS NOT NULL
290
- ORDER BY t.fact_id DESC LIMIT 1`);
291
- let newest;
292
- for (const spelling of (0, scope_1.scopeSpellings)((0, scope_1.normalizeScope)(scope))) {
293
- const row = read.get(spelling);
294
- if (row && (!newest || row.id > newest.id))
295
- newest = row;
296
- }
297
- return newest?.eventId;
298
- }
299
- /** One current lifecycle fact by its row id; undefined once it is superseded (or was never a lifecycle fact). */
300
- function currentFactById(db, id) {
301
- return (0, statements_1.prepareOnce)(db, `SELECT ${FACT_COLUMNS}
302
- FROM facts f JOIN entities e ON e.id = f.subject_entity_id
303
- WHERE f.id = ? AND f.valid_to IS NULL AND f.source_path = '_mcp_'`).get(id);
304
- }
305
- /**
306
- * The newest current facts, for a read with no query. The partial indexes (migration 14) keep them
307
- * in order, so this reads `limit` rows however many facts there are; `INDEXED BY` states that
308
- * intent, because the planner would otherwise pick the source_path index and sort every fact.
309
- */
310
- function recentCurrentFacts(db, scope, limit) {
311
- const read = db.prepare(`SELECT ${FACT_COLUMNS}, f.id AS id
312
- FROM facts f INDEXED BY ${scope === undefined ? 'idx_facts_lifecycle_recent' : 'idx_facts_lifecycle_scope_recent'}
313
- JOIN entities e ON e.id = f.subject_entity_id
314
- WHERE f.valid_to IS NULL AND f.source_path = '_mcp_' ${scope === undefined ? '' : 'AND f.scope = ?'}
315
- ORDER BY f.valid_from DESC, f.id DESC
316
- LIMIT ?`);
317
- const rows = scope === undefined
318
- ? read.all(limit)
319
- : newestOfSpellings(read, scope, limit);
320
- return rows.map(({ id: _id, ...fact }) => fact); // the row id only orders the merge
321
- }