renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,205 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.MAX_DRIFT_VALUES = void 0;
4
- exports.nameOf = nameOf;
5
- exports.related = related;
6
- exports.possibleDuplicates = possibleDuplicates;
7
- exports.namedConflicts = namedConflicts;
8
- const rowStore_1 = require("../context/rowStore");
9
- const queryHelpers_1 = require("../retrieval/queryHelpers");
10
- const factTerms_1 = require("./factTerms");
11
- const scope_1 = require("../mvp/scope");
12
- const sourceName_1 = require("./sourceName");
13
- /**
14
- * Multi-agent drift: current lifecycle facts that may be one thing, named, never merged.
15
- *
16
- * A correction closes only the current fact of the same subject (one identity: case and Unicode form folded),
17
- * predicate and scope (any spelling of it, scopeSpellings). When several agents write, the same thing gets saved under another name (Codex writes
18
- * "Atlas project", Claude Code writes "Atlas") or in another scope (a correction saved without the project's scope
19
- * goes to global), and then both values stay current. This module supersedes nothing across subjects or scopes.
20
- * It names what may be one thing, so the agent or the user can decide:
21
- *
22
- * - `possibleDuplicates`, on a fact save's receipt: the other current facts related to the one just written;
23
- * - `conflicts`, on a read: groups of related current facts whose values disagree, for the facts the read returns.
24
- *
25
- * Two facts are related when they have the same predicate and their subjects are one identity (`identityKey`), or
26
- * one name is the other with one more word before or after it: "Atlas" and "Atlas project", "staging database" and
27
- * "Atlas staging database". A name longer by more words ("Atlas service shipments") or a possessive ("Ravi's
28
- * manager", "user's mother") names another thing. Values are compared after `normDedupeValue`, so "Sao Paulo" and
29
- * "São Paulo" do not disagree.
30
- *
31
- * Scope stays an isolation boundary. A call made in a scope sees that scope's facts only, as its reads do
32
- * (`scopeFilterFor`), so it is never told what another scope holds. A call without a scope sees every scope, so it
33
- * is the one told that a subject and predicate have another current value elsewhere.
34
- *
35
- * Cost: one pass over the current lifecycle facts that reads the subject names of those with an asked predicate, then
36
- * the facts whose name passes a word test in full. Measured with 12 asked facts, 6 of them sharing their predicate with
37
- * a quarter of the facts (host load about 45): 0.4 ms at 150 current facts, 2.4 ms at 2,000, 17 ms at 20,000. Nothing
38
- * is written and nothing is embedded.
39
- */
40
- /** The most values a conflict, or a receipt's duplicates, lists: the newest first, with the total when there are more. */
41
- exports.MAX_DRIFT_VALUES = 8;
42
- /** A folded word that ends in an apostrophe and s ("ravi's", "ravi’s") or in s and an apostrophe ("users'"). */
43
- const POSSESSIVE = /[\p{L}\p{N}](?:['’]s|s['’])(?=\s|$)/u;
44
- /** The fold identityKey applies before its alias table: NFC, trimmed, lower-cased. */
45
- function fold(name) {
46
- return name.normalize('NFC').trim().toLowerCase().normalize('NFC');
47
- }
48
- /** What separates two words of a folded name. A word keeps its hyphens and underscores: "atl-412" is one word. */
49
- const NOT_A_WORD = /[^\p{L}\p{N}\p{M}_-]+/u;
50
- /** The words of a subject name, by name: a read reads the name of every current fact with an asked predicate, and the names are the same read after read. */
51
- const wordsOfName = new Map();
52
- const MAX_WORDS_OF_NAME = 8192;
53
- function wordsOf(subject) {
54
- let words = wordsOfName.get(subject);
55
- if (words === undefined) {
56
- const folded = fold(subject);
57
- const split = POSSESSIVE.test(folded) ? [] : folded.split(NOT_A_WORD).filter(Boolean);
58
- words = split.length > 0 ? split : null;
59
- if (wordsOfName.size >= MAX_WORDS_OF_NAME)
60
- wordsOfName.clear();
61
- wordsOfName.set(subject, words);
62
- }
63
- return words;
64
- }
65
- function nameOf(subject, names) {
66
- let name = names.get(subject);
67
- if (!name) {
68
- name = { subject, words: wordsOf(subject) };
69
- names.set(subject, name);
70
- }
71
- return name;
72
- }
73
- /** Whether `longer` is `shorter` plus one word at its start or its end. */
74
- function oneWordLonger(longer, shorter) {
75
- if (longer.length !== shorter.length + 1)
76
- return false;
77
- const same = (offset) => shorter.every((word, i) => longer[i + offset] === word);
78
- return same(0) || same(1);
79
- }
80
- function related(a, b) {
81
- if ((a.key ??= (0, queryHelpers_1.identityKey)(a.subject)) === (b.key ??= (0, queryHelpers_1.identityKey)(b.subject)))
82
- return true;
83
- if (!a.words || !b.words)
84
- return false;
85
- return oneWordLonger(a.words, b.words) || oneWordLonger(b.words, a.words);
86
- }
87
- /**
88
- * The current lifecycle facts, in one scope or (null) every scope, newest first, that have the predicate of an asked
89
- * fact and a subject that may be related to its subject; `related` then decides. A related name has the asked name's
90
- * first word first, or its last word last: it is the same words, or one more, or one fewer at its start or end. So
91
- * one pass reads the subject name of every current fact with an asked predicate, and only the facts whose name passes
92
- * that test are read in full. A possessive asked name is related by identity alone, so it is looked for whole; an
93
- * alias is looked for by each of its spellings. The partial index of current lifecycle facts holds the predicate, so
94
- * superseded facts are never visited: `INDEXED BY` states it, because the planner would otherwise take the
95
- * source_path index, which holds every lifecycle fact ever saved.
96
- */
97
- function candidates(db, asked, scope, names) {
98
- const firsts = new Set();
99
- const lasts = new Set();
100
- const wholes = new Set();
101
- for (const { subject } of asked) {
102
- for (const spelling of (0, queryHelpers_1.identityNames)(subject)) {
103
- const { words } = nameOf(spelling, names);
104
- if (words) {
105
- firsts.add(words[0]);
106
- lasts.add(words[words.length - 1]);
107
- }
108
- else {
109
- wholes.add(fold(spelling));
110
- }
111
- }
112
- }
113
- const inScope = scope === null ? null : (0, scope_1.savedScopeSqlPredicate)((0, scope_1.normalizeScope)(scope), 'f.scope'); // any spelling of it
114
- const named = (0, factTerms_1.prepareOnce)(db, `SELECT f.id, e.canonical_name
115
- FROM facts f INDEXED BY idx_facts_current
116
- JOIN entities e ON e.id = f.subject_entity_id
117
- WHERE f.valid_to IS NULL AND f.source_path = '_mcp_'
118
- AND f.predicate IN (SELECT value FROM json_each(?))${inScope ? `
119
- AND ${inScope.sql}` : ''}`).raw().all(JSON.stringify([...new Set(asked.map(fact => fact.predicate))]), ...(inScope?.params ?? []));
120
- const ids = named.filter(([, subject]) => {
121
- const { words } = nameOf(subject, names);
122
- return words ? firsts.has(words[0]) || lasts.has(words[words.length - 1]) : wholes.has(fold(subject));
123
- }).map(([id]) => id);
124
- if (ids.length === 0)
125
- return [];
126
- return (0, factTerms_1.prepareOnce)(db, `SELECT f.id AS id, e.canonical_name AS subject, f.predicate AS predicate, COALESCE(f.object_text, '') AS object,
127
- f.scope AS scope, f.valid_from AS validFrom, f.author AS author
128
- FROM json_each(?) c
129
- CROSS JOIN facts f ON f.id = c.value
130
- JOIN entities e ON e.id = f.subject_entity_id
131
- ORDER BY f.valid_from DESC, f.id DESC`).all(JSON.stringify(ids));
132
- }
133
- /** A value as a read shows it: who saved it, when it is another agent's or an import's (memory/sourceName.ts). */
134
- function shown(row, reader) {
135
- return { subject: row.subject, object: row.object, scope: row.scope, validFrom: row.validFrom, ...(0, sourceName_1.authorMarker)(row.author, reader) };
136
- }
137
- /**
138
- * The receipt of a fact save: the other current facts with its predicate whose subject is related to the one written,
139
- * newest first. `factId` is the fact the save left current (it is not its own duplicate); `scope` is the save's scope
140
- * filter (null when the call named no scope).
141
- */
142
- function possibleDuplicates(db, written, factId, scope, reader) {
143
- const names = new Map();
144
- const asked = nameOf(written.subject, names);
145
- const found = candidates(db, [written], scope, names)
146
- .filter(row => row.id !== factId && related(asked, nameOf(row.subject, names)));
147
- return { values: found.slice(0, exports.MAX_DRIFT_VALUES).map(row => shown(row, reader)), total: found.length };
148
- }
149
- /**
150
- * The conflicts behind the current lifecycle facts a read returns: for each of them, the current facts related to it
151
- * (its own row included); every group that holds two different values is one conflict. Groups that share a fact are
152
- * one conflict. Conflicts come in the order of the first fact that names them. `scope` is the read's scope filter
153
- * (null when the read named no scope).
154
- */
155
- function namedConflicts(db, facts, scope, reader) {
156
- if (facts.length === 0)
157
- return [];
158
- const names = new Map();
159
- const rows = candidates(db, facts, scope, names);
160
- if (rows.length < 2)
161
- return [];
162
- const parent = rows.map((_, i) => i);
163
- const root = (i) => {
164
- while (parent[i] !== i)
165
- i = parent[i] = parent[parent[i]];
166
- return i;
167
- };
168
- const matched = new Set(); // rows related to a fact the read returned; only these are ever joined
169
- const firstOfEach = []; // one row of each returned fact's group, in the order the read returned the facts
170
- for (const fact of facts) {
171
- const asked = nameOf(fact.subject, names);
172
- let first = -1;
173
- rows.forEach((row, i) => {
174
- if (row.predicate !== fact.predicate || !related(asked, nameOf(row.subject, names)))
175
- return;
176
- matched.add(i);
177
- if (first < 0)
178
- first = i;
179
- else
180
- parent[root(i)] = root(first);
181
- });
182
- if (first >= 0)
183
- firstOfEach.push(first);
184
- }
185
- const groups = new Map();
186
- for (const i of [...matched].sort((a, b) => a - b)) { // row order: newest first
187
- const group = groups.get(root(i));
188
- if (group)
189
- group.push(rows[i]);
190
- else
191
- groups.set(root(i), [rows[i]]);
192
- }
193
- const conflicts = [];
194
- for (const r of new Set(firstOfEach.map(root))) {
195
- const group = groups.get(r);
196
- if (new Set(group.map(row => (0, rowStore_1.normDedupeValue)(row.object))).size < 2)
197
- continue;
198
- conflicts.push({
199
- predicate: group[0].predicate,
200
- values: group.slice(0, exports.MAX_DRIFT_VALUES).map(row => shown(row, reader)),
201
- ...(group.length > exports.MAX_DRIFT_VALUES ? { total: group.length } : {}),
202
- });
203
- }
204
- return conflicts;
205
- }
@@ -1,321 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.MAX_COMMON_WORDS = exports.RECENT_WINDOW = exports.COMPLETE_POSTINGS = exports.prepareOnce = void 0;
4
- exports.words = words;
5
- exports.atoms = atoms;
6
- exports.atomSequence = atomSequence;
7
- exports.canonicalSubject = canonicalSubject;
8
- exports.factVocabulary = factVocabulary;
9
- exports.indexFactTerms = indexFactTerms;
10
- exports.unindexFactTerms = unindexFactTerms;
11
- exports.ensureFactTerms = ensureFactTerms;
12
- exports.statementTerms = statementTerms;
13
- exports.findCurrentFacts = findCurrentFacts;
14
- exports.newestFactEvent = newestFactEvent;
15
- exports.currentFactById = currentFactById;
16
- exports.recentCurrentFacts = recentCurrentFacts;
17
- const statements_1 = require("../db/statements");
18
- Object.defineProperty(exports, "prepareOnce", { enumerable: true, get: function () { return statements_1.prepareOnce; } });
19
- const scope_1 = require("../mvp/scope");
20
- /**
21
- * The words a lifecycle fact is found by, and the index that finds facts by them.
22
- *
23
- * `fact_terms` is an inverted index over the facts table: one row per (word, scope, fact) for every
24
- * lifecycle fact, current or superseded, written by the transaction that writes the fact. A search
25
- * reads the rows of the query's words and joins only the facts they name, so a fact is found by its
26
- * words however old it is, and the work a read does is bounded (below), not a function of the table.
27
- *
28
- * How far a read goes down a word's rows. A word is *selective* when at most COMPLETE_POSTINGS rows
29
- * of the scope read have it (the rows of superseded facts count): its rows are read completely, so
30
- * every current fact that has it is found, however old, and is counted exactly, however many other
31
- * selective words the query has. A word with more rows is *common*: its rows are not read, and the
32
- * facts it finds are those among the RECENT_WINDOW newest current facts of the scope. So a table of
33
- * at most COMPLETE_POSTINGS facts is always read exactly, and so is any query with a selective word.
34
- * What is given up, and all of it: a fact that has none of the query's selective words and is older
35
- * than the newest RECENT_WINDOW facts is not found through common words alone, and an any-words read
36
- * ranks by at most MAX_COMMON_WORDS common words. Every candidate is ranked by the number of the
37
- * query's words it has.
38
- *
39
- * The work is bounded by the query, not by the table: a word costs at most COMPLETE_POSTINGS + 1
40
- * rows to classify, and a read collects at most COMPLETE_POSTINGS facts per selective word, so at
41
- * most (words in the query) x COMPLETE_POSTINGS candidates, plus RECENT_WINDOW; the tools cap a query
42
- * at 1,000 characters. Nothing is dropped to keep it so.
43
- *
44
- * It is derived data, like the rest of index.db: `ensureFactTerms` indexes whatever is missing (a
45
- * database from before the index, or one a release without it has written to). Rows are never
46
- * removed: a superseded fact keeps its rows, and the join to `facts` filters it out as current.
47
- *
48
- * No code deletes a lifecycle fact (a correction closes it). Code that ever does must delete its
49
- * rows here too, or the words of an erased fact stay in index.db. The table has no foreign key to
50
- * `facts` on purpose: deleting a note's facts would then scan this whole table once per fact.
51
- */
52
- /** The words of `value` in the order they occur, repeats included. */
53
- function tokens(value) {
54
- return value
55
- .toLowerCase()
56
- .replace(/[^\p{L}\p{N}\p{M}_-]+/gu, ' ')
57
- .split(/\s+/)
58
- .map(w => w.trim())
59
- .filter(w => w.length >= 2);
60
- }
61
- function words(value) {
62
- return Array.from(new Set(tokens(value)));
63
- }
64
- /**
65
- * Like `words`, but slugs and snake_case split into their parts, the way the full-text index
66
- * reads them: `renor-memory-1002` is renor, memory and 1002.
67
- */
68
- function atoms(value) {
69
- return Array.from(new Set(atomSequence(value)));
70
- }
71
- /** `atoms` in the order they occur, repeats included: which words are next to each other. */
72
- function atomSequence(value) {
73
- return tokens(value.replace(/[-_]+/g, ' '));
74
- }
75
- function predicateWords(predicate) {
76
- const out = new Set();
77
- const irregularBases = {
78
- does: 'do',
79
- has: 'have',
80
- };
81
- for (const word of atoms(predicate)) {
82
- out.add(word);
83
- // Relation keys commonly use third-person singular verbs while users ask
84
- // with the base form. Keep this intentionally bounded so predicate nouns
85
- // such as "status" do not get arbitrary stems.
86
- const irregular = irregularBases[word];
87
- if (irregular) {
88
- out.add(irregular);
89
- }
90
- else if (/^[a-z]+ies$/.test(word) && word.length >= 5) {
91
- out.add(`${word.slice(0, -3)}y`);
92
- }
93
- else if (/^[a-z]+(ches|shes|xes|zes|oes|sses)$/.test(word)) {
94
- out.add(word.slice(0, -2));
95
- }
96
- else if (/^[a-z]+s$/.test(word)
97
- && word.length >= 4
98
- && !word.endsWith('ss')
99
- && !word.endsWith('us')
100
- && !word.endsWith('is')) {
101
- out.add(word.slice(0, -1));
102
- }
103
- }
104
- return Array.from(out);
105
- }
106
- /** The name a read shows for a subject: the entity's, which may be an earlier spelling of an event's. */
107
- function canonicalSubject(db, entityId) {
108
- return (0, statements_1.prepareOnce)(db, 'SELECT canonical_name FROM entities WHERE id = ?').get(entityId).canonical_name;
109
- }
110
- /** Every word a fact is found by: its subject and object, and its predicate with common verb forms. */
111
- function factVocabulary(fact) {
112
- return new Set([...atoms(`${fact.subject} ${fact.object}`), ...predicateWords(fact.predicate)]);
113
- }
114
- /**
115
- * Indexes one fact. The empty term is a marker, not a word (a query word is never empty): it says
116
- * the fact has been indexed, so one whose text has no words is not indexed again and again.
117
- */
118
- function indexFactTerms(db, factId, scope, fact) {
119
- (0, statements_1.prepareOnce)(db, `INSERT OR IGNORE INTO fact_terms (term, scope, fact_id) SELECT value, ?, ? FROM json_each(?)`)
120
- .run(scope, factId, JSON.stringify(['', ...factVocabulary(fact)]));
121
- }
122
- /** Removes what indexFactTerms wrote for one fact (a retired fact: memory/lifecycle.ts). */
123
- function unindexFactTerms(db, factId, scope, fact) {
124
- (0, statements_1.prepareOnce)(db, `DELETE FROM fact_terms WHERE scope = ? AND fact_id = ? AND term IN (SELECT value FROM json_each(?))`)
125
- .run(scope, factId, JSON.stringify(['', ...factVocabulary(fact)]));
126
- }
127
- /** Indexes every lifecycle fact that is not indexed yet, in one transaction; returns how many. */
128
- function ensureFactTerms(db) {
129
- const unindexed = db.prepare(`SELECT f.id AS id, f.scope AS scope, e.canonical_name AS subject, f.predicate AS predicate, f.object_text AS object
130
- FROM facts f JOIN entities e ON e.id = f.subject_entity_id
131
- WHERE f.source_path = '_mcp_'
132
- AND NOT EXISTS (SELECT 1 FROM fact_terms t WHERE t.term = '' AND t.scope = f.scope AND t.fact_id = f.id)`).all();
133
- if (unindexed.length === 0)
134
- return 0;
135
- db.transaction(() => {
136
- for (const fact of unindexed)
137
- indexFactTerms(db, fact.id, fact.scope, { ...fact, object: fact.object ?? '' });
138
- })();
139
- return unindexed.length;
140
- }
141
- /**
142
- * The words of a fact's saved statement, kept with the fact (`facts.statement_words`) so the facts lane can count them
143
- * toward a question's words (agentMemory.ts `rankCandidateFacts`). The atoms in the order they occur, space-joined: an
144
- * atom never holds a space. Null when the fact was saved with no statement (save_memory) or has no word in it.
145
- */
146
- function statementTerms(statement) {
147
- if (statement === undefined)
148
- return null;
149
- const terms = atoms(statement);
150
- return terms.length > 0 ? terms.join(' ') : null;
151
- }
152
- const FACT_COLUMNS = `f.event_id AS eventId, e.canonical_name AS subject, f.predicate AS predicate,
153
- f.object_text AS object, f.valid_from AS validFrom, f.scope AS scope, f.author AS author,
154
- f.statement_words AS statementWords`;
155
- const FACT_MATCH_COLUMNS = `f.id AS id, ${FACT_COLUMNS}`;
156
- /** A word with at most this many rows (in the scope read) is selective: its rows are read completely. */
157
- exports.COMPLETE_POSTINGS = 1_000;
158
- /** A common word is matched among this many of the newest current facts of the scope. */
159
- exports.RECENT_WINDOW = 500;
160
- /** An any-words read ranks by at most this many common words (the first in the query); selective words all count. */
161
- exports.MAX_COMMON_WORDS = 8;
162
- /**
163
- * The query's words that some fact has, selective ones first and the rarest of them first. Reads at
164
- * most COMPLETE_POSTINGS + 1 rows of a word, which is enough to know it is common. With `needAll`,
165
- * a word no fact has ends the read: nothing can have every word.
166
- */
167
- function readWords(db, terms, scope, needAll) {
168
- const inScope = scope === undefined ? undefined : (0, scope_1.savedScopeSqlPredicate)((0, scope_1.normalizeScope)(scope), 'scope');
169
- const read = (0, statements_1.prepareOnce)(db, `SELECT fact_id FROM fact_terms WHERE term = ? ${inScope === undefined ? '' : `AND ${inScope.sql}`} LIMIT ?`).pluck();
170
- const found = [];
171
- for (const term of terms) {
172
- const ids = read.all(term, ...(inScope?.params ?? []), exports.COMPLETE_POSTINGS + 1);
173
- if (ids.length === 0) {
174
- if (needAll)
175
- return null;
176
- continue;
177
- }
178
- const common = ids.length > exports.COMPLETE_POSTINGS;
179
- found.push({ term, ids: common ? [] : ids, common });
180
- }
181
- return found.sort((a, b) => Number(a.common) - Number(b.common) || a.ids.length - b.ids.length);
182
- }
183
- /**
184
- * The rows `read` returns for each spelling of `scope` (scopeSpellings), merged newest first and cut to `limit`.
185
- * `read` takes a spelling and a limit and returns at most that many rows of a partial index, newest first, so the
186
- * merge never sorts more than one `limit` per spelling, and a scope with one spelling (global) is returned as read.
187
- */
188
- function newestOfSpellings(read, scope, limit) {
189
- const spellings = (0, scope_1.scopeSpellings)((0, scope_1.normalizeScope)(scope));
190
- const rows = spellings.flatMap(spelling => read.all(spelling, limit));
191
- if (spellings.length === 1)
192
- return rows;
193
- return rows
194
- .sort((a, b) => (a.validFrom === b.validFrom ? b.id - a.id : a.validFrom < b.validFrom ? 1 : -1))
195
- .slice(0, limit);
196
- }
197
- /** The ids of the newest current facts of the scope: `RECENT_WINDOW` rows of a partial index per spelling, in order. */
198
- function recentFactIds(db, scope) {
199
- const read = (0, statements_1.prepareOnce)(db, `SELECT f.id AS id, f.valid_from AS validFrom FROM facts f INDEXED BY ${scope === undefined ? 'idx_facts_lifecycle_recent' : 'idx_facts_lifecycle_scope_recent'}
200
- WHERE f.valid_to IS NULL AND f.source_path = '_mcp_' ${scope === undefined ? '' : 'AND f.scope = ?'}
201
- ORDER BY f.valid_from DESC, f.id DESC LIMIT ?`);
202
- const rows = scope === undefined
203
- ? read.all(exports.RECENT_WINDOW)
204
- : newestOfSpellings(read, scope, exports.RECENT_WINDOW);
205
- return rows.map(row => row.id);
206
- }
207
- /**
208
- * The current facts found by `terms`, best first: more of the words, then newest. `scope` is one
209
- * scope when given (a normalized scope, read under every spelling of it: scopeSpellings) and every
210
- * scope when omitted. Only current lifecycle facts count.
211
- *
212
- * `any` ranks the facts that have at least one word; `all` needs every word. How much of a word's
213
- * rows is read, and what that gives up, is in the header of this file.
214
- */
215
- function findCurrentFacts(db, args) {
216
- const { scope, match, limit } = args;
217
- const terms = [...new Set(args.terms)].filter(term => term !== ''); // '' is the marker row, never a word
218
- if (terms.length === 0 || limit < 1)
219
- return [];
220
- const words = readWords(db, terms, scope, match === 'all');
221
- if (!words || words.length === 0)
222
- return [];
223
- const scoped = scope === undefined ? undefined : (0, scope_1.savedScopeSqlPredicate)((0, scope_1.normalizeScope)(scope), 'f.scope');
224
- const inScope = scoped === undefined ? '' : `AND ${scoped.sql}`;
225
- const scopeArgs = scoped?.params ?? [];
226
- if (match === 'any') {
227
- // Every selective word is read completely; only the common words are matched through the newest
228
- // facts, and at most MAX_COMMON_WORDS of them are probed.
229
- const probed = words.filter(word => word.common).slice(0, exports.MAX_COMMON_WORDS);
230
- // Fact id -> how many selective words it has, counted from the rows already read.
231
- const candidates = new Map();
232
- for (const word of words)
233
- if (!word.common)
234
- for (const id of word.ids)
235
- candidates.set(id, (candidates.get(id) ?? 0) + 1);
236
- if (probed.length > 0)
237
- for (const id of recentFactIds(db, scope))
238
- if (!candidates.has(id))
239
- candidates.set(id, 0);
240
- // The candidates are ranked on their id, date and count alone; only the `limit` best are joined to
241
- // their entity and read in full. A window fact that has none of the probed words scores 0 and is dropped.
242
- return (0, statements_1.prepareOnce)(db, `WITH scored AS MATERIALIZED (
243
- SELECT f.id AS id, f.valid_from AS validFrom,
244
- json_extract(c.value, '$[1]')${probed.length === 0 ? '' : `
245
- + (SELECT COUNT(*) FROM fact_terms p
246
- WHERE p.term IN (${probed.map(() => '?').join(', ')}) AND p.scope = f.scope AND p.fact_id = f.id)`} AS shared
247
- FROM json_each(?) c
248
- CROSS JOIN facts f ON f.id = json_extract(c.value, '$[0]')
249
- WHERE f.valid_to IS NULL AND f.source_path = '_mcp_' ${inScope}
250
- ), best AS (
251
- SELECT id, validFrom, shared FROM scored WHERE shared > 0
252
- ORDER BY shared DESC, validFrom DESC, id DESC
253
- LIMIT ?
254
- )
255
- SELECT ${FACT_MATCH_COLUMNS}, b.shared AS shared
256
- FROM best b
257
- JOIN facts f ON f.id = b.id
258
- JOIN entities e ON e.id = f.subject_entity_id
259
- ORDER BY b.shared DESC, b.validFrom DESC, b.id DESC`).all(...probed.map(word => word.term), JSON.stringify([...candidates]), ...scopeArgs, limit);
260
- }
261
- // `all`: every fact that has every word has the rarest selective word, so its rows are the
262
- // candidates and the other words are probed, one candidate at a time. With no selective word, the
263
- // candidates are the newest facts and every word is probed.
264
- const driver = words.find(word => !word.common);
265
- const candidates = driver ? driver.ids : recentFactIds(db, scope);
266
- const probed = words.filter(word => word !== driver).map(word => word.term);
267
- return (0, statements_1.prepareOnce)(db, `SELECT ${FACT_MATCH_COLUMNS}, ${terms.length} AS shared
268
- FROM json_each(?) c
269
- CROSS JOIN facts f ON f.id = c.value
270
- JOIN entities e ON e.id = f.subject_entity_id
271
- WHERE f.valid_to IS NULL AND f.source_path = '_mcp_' ${inScope}
272
- ${probed.map(() => 'AND EXISTS (SELECT 1 FROM fact_terms p WHERE p.term = ? AND p.scope = f.scope AND p.fact_id = f.id)').join('\n ')}
273
- ORDER BY f.valid_from DESC, f.id DESC
274
- LIMIT ?`).all(JSON.stringify(candidates), ...scopeArgs, ...probed, limit);
275
- }
276
- /**
277
- * The ledger event that wrote the newest lifecycle fact (current or superseded) of one scope, or of
278
- * every scope when none is given. A scope's newest fact is the last row of its marker in the term
279
- * index, one seek per spelling of the scope (scopeSpellings); without a scope it is the last row of
280
- * the facts table.
281
- */
282
- function newestFactEvent(db, scope) {
283
- if (scope === undefined) {
284
- const row = (0, statements_1.prepareOnce)(db, `SELECT event_id AS eventId FROM facts WHERE source_path = '_mcp_' AND event_id IS NOT NULL ORDER BY id DESC LIMIT 1`).get();
285
- return row?.eventId;
286
- }
287
- const read = (0, statements_1.prepareOnce)(db, `SELECT t.fact_id AS id, f.event_id AS eventId
288
- FROM fact_terms t CROSS JOIN facts f ON f.id = t.fact_id
289
- WHERE t.term = '' AND t.scope = ? AND f.source_path = '_mcp_' AND f.event_id IS NOT NULL
290
- ORDER BY t.fact_id DESC LIMIT 1`);
291
- let newest;
292
- for (const spelling of (0, scope_1.scopeSpellings)((0, scope_1.normalizeScope)(scope))) {
293
- const row = read.get(spelling);
294
- if (row && (!newest || row.id > newest.id))
295
- newest = row;
296
- }
297
- return newest?.eventId;
298
- }
299
- /** One current lifecycle fact by its row id; undefined once it is superseded (or was never a lifecycle fact). */
300
- function currentFactById(db, id) {
301
- return (0, statements_1.prepareOnce)(db, `SELECT ${FACT_COLUMNS}
302
- FROM facts f JOIN entities e ON e.id = f.subject_entity_id
303
- WHERE f.id = ? AND f.valid_to IS NULL AND f.source_path = '_mcp_'`).get(id);
304
- }
305
- /**
306
- * The newest current facts, for a read with no query. The partial indexes (migration 14) keep them
307
- * in order, so this reads `limit` rows however many facts there are; `INDEXED BY` states that
308
- * intent, because the planner would otherwise pick the source_path index and sort every fact.
309
- */
310
- function recentCurrentFacts(db, scope, limit) {
311
- const read = db.prepare(`SELECT ${FACT_COLUMNS}, f.id AS id
312
- FROM facts f INDEXED BY ${scope === undefined ? 'idx_facts_lifecycle_recent' : 'idx_facts_lifecycle_scope_recent'}
313
- JOIN entities e ON e.id = f.subject_entity_id
314
- WHERE f.valid_to IS NULL AND f.source_path = '_mcp_' ${scope === undefined ? '' : 'AND f.scope = ?'}
315
- ORDER BY f.valid_from DESC, f.id DESC
316
- LIMIT ?`);
317
- const rows = scope === undefined
318
- ? read.all(limit)
319
- : newestOfSpellings(read, scope, limit);
320
- return rows.map(({ id: _id, ...fact }) => fact); // the row id only orders the merge
321
- }