renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,195 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ADVICE_LINES_HEADER = exports.MAX_ADVICE_CANDIDATES = exports.MAX_ADVICE_LINES_PER_SESSION = exports.MAX_ADVICE_TOKENS = exports.MAX_ADVICE_LINES = void 0;
4
- exports.readerAdviceLinesEnabled = readerAdviceLinesEnabled;
5
- exports.isSelfStatement = isSelfStatement;
6
- exports.adviceLines = adviceLines;
7
- exports.withAdviceLines = withAdviceLines;
8
- const tokenizer_1 = require("../indexer/tokenizer");
9
- const adviceQuestion_1 = require("./adviceQuestion");
10
- const orchestrator_1 = require("./orchestrator");
11
- const readerEvents_1 = require("./readerEvents");
12
- const relativeTimeHints_1 = require("./relativeTimeHints");
13
- /**
14
- * Advice lines: for a query that asks to be advised ("Can you recommend ...", "Any tips ...", "What should I ..."; adviceQuestion.ts),
15
- * one block first with what the user has said about their own situation, tastes and habits that shares words with the request,
16
- * quoted as they wrote it and dated by its conversation, oldest first. Opt-in (`RENOR_READER_ADVICE_LINES`, default off): with it
17
- * unset, or set to anything but an on value, a reply is byte-identical, and so is the reply to any query that does not ask for advice.
18
- *
19
- * Why. On the 19 Oct best set with a Claude Haiku 5.5 reader (docs/evals/results/LEDGER.md section AG), 30 of the 61 misses are
20
- * "I don't know" on a question that has an answer; 28 of those 30 pages hold every gold evidence turn, and the largest group of them
21
- * is 10 requests for advice (preference questions), each wrong in all six arms on record whatever the page's layout or digests
22
- * (docs/evals/results/idk-misses-20261019/). A request for advice has no stored answer: what memory holds is the user's earlier words
23
- * about themselves ("I've been using Premiere Pro for my edits"), one sentence inside a long turn among dozens of excerpts. These
24
- * lines put those sentences where a reader looks first and say what they are.
25
- *
26
- * - A line is `<day>: "<sentence>"`, the sentence exactly as the user wrote it (one over MAX_QUOTE_CHARS is cut around its first
27
- * matching word, with "…" where it is cut), under the date of its conversation. Assistant turns are not quoted, and neither is a
28
- * question.
29
- * - Picked by the nouns and names the request holds, not by its advice words ("recommend", "tips") or its verbs: one of them is
30
- * enough. Every thing the request names gets its own line first, then the best-scoring of the rest (rarer words weigh more).
31
- * - Bounded: at most MAX_ADVICE_LINES lines, two per conversation, MAX_ADVICE_TOKENS tokens of lines; only conversations the page
32
- * already holds are read; no model call, no embedding.
33
- * - The excerpts stay in full below the lines, in the order the page gave them.
34
- */
35
- /** Default off. `1` / `true` / `yes` / `on` places the advice lines first on the reply to a query that asks to be advised. */
36
- function readerAdviceLinesEnabled(env = process.env) {
37
- const v = (env.RENOR_READER_ADVICE_LINES ?? '').trim().toLowerCase();
38
- return v === '1' || v === 'true' || v === 'yes' || v === 'on';
39
- }
40
- /** Most lines a reply carries. */
41
- exports.MAX_ADVICE_LINES = 6;
42
- /** Most tokens of lines (o200k) a reply carries, the header sentence excluded. */
43
- exports.MAX_ADVICE_TOKENS = 400;
44
- /** Most lines taken from one conversation. */
45
- exports.MAX_ADVICE_LINES_PER_SESSION = 2;
46
- /** Most sentences considered for one query: the cost of a reply is bounded whatever the page holds. */
47
- exports.MAX_ADVICE_CANDIDATES = 4000;
48
- exports.ADVICE_LINES_HEADER = 'Statements by the user about their own situation, tastes and habits that share words with this request, oldest first, each under the date of its conversation. The excerpts below are unchanged.';
49
- // The words a request for advice is framed with, not what it is about: they never select a sentence.
50
- const ADVICE_WORDS = new Set([
51
- 'recommend', 'recommendation', 'suggest', 'suggestion', 'advise', 'advice', 'tip', 'idea', 'thought', 'pointer', 'opinion', 'think',
52
- 'option', 'resource', 'good', 'great', 'best', 'better', 'nice', 'new', 'like', 'something', 'anything', 'thing', 'way', 'please',
53
- 'should', 'want', 'need', 'looking', 'help', 'know', 'let',
54
- ].map(w => (0, readerEvents_1.stem)(w)));
55
- // A question to someone, not something the user says about themselves.
56
- const ASKS = /^(?:can|could|would|will|do|does|did|is|are|what|how|why|where|which|who|any)\b/i;
57
- /** Something the user says about themselves: first person, and not a question. */
58
- function isSelfStatement(sentence) {
59
- const s = sentence.replace(/^["'“(\[]+/, '');
60
- return !sentence.trimEnd().endsWith('?') && !ASKS.test(s) && readerEvents_1.FIRST_PERSON.test(s);
61
- }
62
- /** The conversation's own day, as its date is written (`2023/05/30 (Tue)` or `2023-05-30`), or null when it has none. */
63
- function dayOf(sessionDate) {
64
- const ms = (0, orchestrator_1.sessionDateMs)(sessionDate);
65
- if (!Number.isFinite(ms) || sessionDate === undefined)
66
- return null;
67
- return (0, relativeTimeHints_1.formatDay)(new Date(Math.floor(ms / 86_400_000) * 86_400_000), /^\d{4}\//.test(sessionDate));
68
- }
69
- /**
70
- * The advice lines for one query, or null when the query does not ask to be advised or no statement of the user's shares a
71
- * word with it. `hits` are the conversation hits of the page; `turnsOf` returns the stored chunks of a conversation note.
72
- */
73
- function adviceLines(hits, query, turnsOf) {
74
- if (!(0, adviceQuestion_1.asksForAdvice)(query))
75
- return null;
76
- const terms = new Set([...(0, readerEvents_1.questionTerms)(query)].filter(t => !ADVICE_WORDS.has(t)));
77
- const aboutTerms = new Set([...terms].filter(t => !readerEvents_1.GENERIC_VERBS.has(t)));
78
- if (aboutTerms.size === 0)
79
- return null;
80
- // One entry per conversation, in the order the page holds them: its date and a chunk of it, from whichever of its hits has them.
81
- const sources = new Map();
82
- for (const hit of hits) {
83
- const had = sources.get(hit.notePath) ?? {};
84
- sources.set(hit.notePath, { chunkId: had.chunkId ?? (hit.chunkId !== undefined && hit.chunkId > 0 ? hit.chunkId : undefined), sessionDate: had.sessionDate ?? hit.session_date });
85
- }
86
- const candidates = [];
87
- let position = 0;
88
- for (const [notePath, { chunkId, sessionDate }] of sources) {
89
- if (notePath === '' || candidates.length >= exports.MAX_ADVICE_CANDIDATES)
90
- continue;
91
- for (const turn of turnsOf({ notePath, chunkId })) {
92
- if (candidates.length >= exports.MAX_ADVICE_CANDIDATES)
93
- break;
94
- if (!(0, readerEvents_1.isUserTurn)(turn.headingPath))
95
- continue;
96
- for (const sentence of (0, readerEvents_1.sentencesOf)((0, readerEvents_1.stripHeading)(turn.text))) {
97
- if (!isSelfStatement(sentence))
98
- continue;
99
- candidates.push({ sentence, stems: new Set((0, readerEvents_1.wordsOf)(sentence).map(w => (0, readerEvents_1.stem)(w.replace(/['’]s$/, '')))), notePath, sessionDate, position: position++ });
100
- }
101
- }
102
- }
103
- if (candidates.length === 0)
104
- return null;
105
- // A term every sentence holds says little; one a few hold says what the request is about.
106
- const df = new Map();
107
- for (const c of candidates)
108
- for (const t of aboutTerms)
109
- if (c.stems.has(t))
110
- df.set(t, (df.get(t) ?? 0) + 1);
111
- const idf = (t) => Math.log(1 + candidates.length / (df.get(t) ?? 1));
112
- // A preference is usually stated with one word of the request ("I like hotels near the beach" for "a hotel for my trip to Miami"),
113
- // so one shared word is enough to be a candidate; the score and the bounds below decide which candidates are shown.
114
- const scored = candidates.flatMap(candidate => {
115
- let score = 0;
116
- for (const t of aboutTerms)
117
- if (candidate.stems.has(t))
118
- score += idf(t);
119
- return score > 0 ? [{ candidate, score }] : [];
120
- });
121
- if (scored.length === 0)
122
- return null;
123
- const chosen = [];
124
- const perSession = new Map();
125
- const seen = new Set();
126
- let tokens = 0;
127
- const take = (c) => {
128
- const key = `${c.notePath}\0${c.sentence.toLowerCase()}`;
129
- if (seen.has(key) || (perSession.get(c.notePath) ?? 0) >= exports.MAX_ADVICE_LINES_PER_SESSION)
130
- return false;
131
- const day = dayOf(c.sessionDate);
132
- const quote = (0, readerEvents_1.quoteOf)(c.sentence, terms);
133
- const line = day ? `- ${day}: "${quote}"` : `- "${quote}"`;
134
- const cost = (0, tokenizer_1.encodeTokens)(line) + 1;
135
- if (tokens + cost > exports.MAX_ADVICE_TOKENS)
136
- return false;
137
- tokens += cost;
138
- seen.add(key);
139
- perSession.set(c.notePath, (perSession.get(c.notePath) ?? 0) + 1);
140
- chosen.push({ candidate: c, line });
141
- return true;
142
- };
143
- // First every thing the request names gets its best line, the sentence that says most of what is still unsaid first; then the
144
- // best-scoring sentences that are left, until a bound is reached.
145
- const covered = new Set();
146
- const refused = new Set();
147
- while (chosen.length < exports.MAX_ADVICE_LINES) {
148
- let pick = null;
149
- let pickGain = 0;
150
- for (const s of scored) {
151
- if (refused.has(s.candidate))
152
- continue;
153
- let gain = 0;
154
- for (const t of aboutTerms)
155
- if (s.candidate.stems.has(t) && !covered.has(t))
156
- gain += idf(t);
157
- if (gain > pickGain || (gain === pickGain && gain > 0 && pick !== null && s.score > pick.score)) {
158
- pick = s;
159
- pickGain = gain;
160
- }
161
- }
162
- if (pick === null || pickGain === 0)
163
- break;
164
- if (take(pick.candidate))
165
- for (const t of aboutTerms) {
166
- if (pick.candidate.stems.has(t))
167
- covered.add(t);
168
- }
169
- refused.add(pick.candidate);
170
- }
171
- const rest = scored.filter(s => !refused.has(s.candidate))
172
- .sort((a, b) => b.score - a.score || a.candidate.sentence.length - b.candidate.sentence.length || a.candidate.position - b.candidate.position);
173
- for (const s of rest) {
174
- if (chosen.length >= exports.MAX_ADVICE_LINES)
175
- break;
176
- take(s.candidate);
177
- }
178
- if (chosen.length === 0)
179
- return null;
180
- // Oldest first: the conversation's date, then the order of reading.
181
- chosen.sort((a, b) => ((0, orchestrator_1.sessionDateMs)(a.candidate.sessionDate) - (0, orchestrator_1.sessionDateMs)(b.candidate.sessionDate)) || (a.candidate.position - b.candidate.position));
182
- return [exports.ADVICE_LINES_HEADER, ...chosen.map(c => c.line)].join('\n');
183
- }
184
- /**
185
- * The page with the advice lines first (reason "advice"), when the switch is on and the query asks to be advised. Otherwise the
186
- * very array it was given.
187
- */
188
- function withAdviceLines(hits, input, env = process.env) {
189
- if (!readerAdviceLinesEnabled(env) || !(0, adviceQuestion_1.asksForAdvice)(input.query))
190
- return hits;
191
- const text = adviceLines(hits.filter(input.isSource), input.query, input.turnsOf());
192
- if (text === null)
193
- return hits;
194
- return [{ notePath: '', text, score: 0, reason: 'advice' }, ...hits];
195
- }
@@ -1,44 +0,0 @@
1
- "use strict";
2
- /**
3
- * Does a search query ask to be advised: for a recommendation, a suggestion, a tip, an idea or an opinion on what to do?
4
- * Told from the query's own words and nothing else (never a label of the question), because the query is all a search sees.
5
- * Its one use is the opt-in exception to the reader layout (readerLayout.ts, RENOR_READER_LAYOUT_SKIP_ADVICE).
6
- *
7
- * Narrow on purpose, since a wrong yes takes the layout from a question that wants it:
8
- *
9
- * - a request is one of a few plain frames ("Can you recommend ...", "Any tips ...", "What should I ...", "What do you
10
- * think"); a word such as "recommendations" inside a phrase that looks something up is not a request;
11
- * - a query that points back at what was said before ("what did you recommend last time", "remind me", "our previous
12
- * conversation") asks to recall an answer, not to be given one, and is not a request wherever the frame stands;
13
- * - only the last line is read for the frame: the benchmark runner puts "Question date: ..." above the question.
14
- */
15
- Object.defineProperty(exports, "__esModule", { value: true });
16
- exports.asksForAdvice = asksForAdvice;
17
- // "any" opens a clause ("Any tips?", "..., any suggestions?") or follows "do you have": "did I get any tips" looks something up.
18
- const ANY_ADVICE = String.raw `any\s+(?:[\w'’-]+\s+){0,3}(?:tips?|advice|suggestions?|recommendations?|ideas|thoughts|pointers)\b`;
19
- const REQUEST_FRAMES = [
20
- /\b(?:can|could|would|will) you (?:please )?(?:recommend|suggest|advise)\b/i,
21
- new RegExp(String.raw `(?:^|[,;:.!?]\s*|\bdo you have\s+)${ANY_ADVICE}`, 'i'),
22
- /\bwhat should i\b/i,
23
- /\b(?:what do you think|do you think|what would you (?:do|recommend|suggest))\b/i,
24
- ];
25
- const POINTS_AT_THE_PAST = new RegExp([
26
- String.raw `\byou (?:recommended|suggested|advised|told|said|mentioned|gave|provided|listed|named)\b`,
27
- String.raw `\blast time\b`,
28
- String.raw `\bearlier\b`,
29
- String.raw `\bremind me\b`,
30
- String.raw `\bwe (?:discussed|talked|chatted|spoke)\b`,
31
- String.raw `\bour (?:previous|last|earlier|past) (?:chat|conversation|discussion)s?\b`,
32
- String.raw `\bprevious(?:ly)?\b`,
33
- String.raw `\bdid you\b`,
34
- ].join('|'), 'i');
35
- function lastLine(query) {
36
- const lines = query.split(/\r?\n/).map(line => line.trim()).filter(line => line !== '');
37
- return lines.length > 0 ? lines[lines.length - 1] : '';
38
- }
39
- function asksForAdvice(query) {
40
- if (POINTS_AT_THE_PAST.test(query))
41
- return false;
42
- const question = lastLine(query);
43
- return REQUEST_FRAMES.some(frame => frame.test(question));
44
- }
@@ -1,44 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.CONVERSATION_MIN_TURNS = void 0;
4
- exports.isTurnLabel = isTurnLabel;
5
- exports.isConversationShaped = isConversationShaped;
6
- /**
7
- * Is a note shaped like a conversation? Told from the note's own sections, never from a front-matter `kind`, a file
8
- * name or a source: a chat somebody exported and a benchmark's sessions look the same, and a note the user wrote as a
9
- * document does not.
10
- *
11
- * A conversation written as Markdown gives each turn its own heading, and the heading says who spoke or which turn it
12
- * is and nothing else ("Turn 7 - User", "User", "### Assistant:", "Q3"). A document's headings name what the section
13
- * is about. So a note is conversation-shaped when at least half of its distinct heading paths end in such a label.
14
- *
15
- * Why the session expansion asks (orchestrator.ts): its statement limit was measured on conversations only. A turn
16
- * over 100 tokens is mostly an assistant's long answer, and the answer-bearing remark is short; in a document the
17
- * paragraph that answers is often 100 to 200 tokens, which the same limit would leave out.
18
- *
19
- * The rule leans to "document". A note taken for a document keeps the former limit, which costs a saving and nothing
20
- * else; a document taken for a conversation would lose paragraphs the limit was never measured on. So a label must be
21
- * made of role words alone, a note needs at least `CONVERSATION_MIN_TURNS` of them, and they must be at least half of its paths.
22
- */
23
- const TURN_WORDS = new Set([
24
- 'turn', 'user', 'assistant', 'human', 'ai', 'bot', 'me', 'you', 'system', 'claude', 'chatgpt', 'gpt', 'gemini',
25
- 'model', 'speaker', 'message', 'question', 'answer', 'reply', 'response', 'prompt', 'q', 'a',
26
- ]);
27
- /** The fewest turn headings a note needs to count as a conversation (a one-exchange note is a document here). */
28
- exports.CONVERSATION_MIN_TURNS = 4;
29
- /** True when the heading is only a speaker or turn label: role words, digits, punctuation, symbols and spaces. */
30
- function isTurnLabel(heading) {
31
- const words = heading.toLowerCase().replace(/[\p{N}\p{P}\p{S}\p{Z}]+/gu, ' ').trim().split(' ').filter(Boolean);
32
- return words.length > 0 && words.every(word => TURN_WORDS.has(word));
33
- }
34
- /** `headingPaths`: the `heading_path` of every chunk of one note (a path is its headings joined by " > "). */
35
- function isConversationShaped(headingPaths) {
36
- const distinct = new Set(headingPaths);
37
- let turns = 0;
38
- for (const headingPath of distinct) {
39
- const last = headingPath.split(' > ').pop() ?? '';
40
- if (isTurnLabel(last))
41
- turns += 1;
42
- }
43
- return turns >= exports.CONVERSATION_MIN_TURNS && turns * 2 >= distinct.size;
44
- }
@@ -1,55 +0,0 @@
1
- "use strict";
2
- var __importDefault = (this && this.__importDefault) || function (mod) {
3
- return (mod && mod.__esModule) ? mod : { "default": mod };
4
- };
5
- Object.defineProperty(exports, "__esModule", { value: true });
6
- exports.createDailyEntryClock = createDailyEntryClock;
7
- const fs_1 = __importDefault(require("fs"));
8
- const dailyNote_1 = require("../memory/dailyNote");
9
- /**
10
- * When the daily-note entry a hit is part of was saved.
11
- *
12
- * A daily note is ONE file holding one entry per statement, so the date of the note (when the file was last written)
13
- * is the same for every entry of the day: a correction said later tied with what it corrects, and newest first could
14
- * put the stale one on top (live dogfood, D3). Each entry is a memory of its own (the chunker cuts one per entry) and
15
- * its header says when it was said: the day of its file, the time, and the offset of the clock it was stamped on. That
16
- * is `entryInstant`, the moment the memory map shows for the entry; an entry filed under an earlier day is dated by
17
- * that day.
18
- *
19
- * The offset is in the file only (readers get the user's own day and time without it), so the file is read, once per
20
- * note however many of its entries are hits, and only for a hit in a daily note. A hit is the entry whose text holds
21
- * it, as the index cut it. A hit that no entry holds, or that entries saved at different moments all hold (the file
22
- * changed since it was indexed, text above the first entry, a passage repeated across entries), keeps its note's date.
23
- */
24
- function createDailyEntryClock() {
25
- const notes = new Map();
26
- return async (hit) => {
27
- const day = (0, dailyNote_1.dailyNoteDayName)(hit.notePath);
28
- if (day === null)
29
- return undefined;
30
- let entries = notes.get(hit.notePath);
31
- if (!entries)
32
- notes.set(hit.notePath, (entries = entriesOf(hit.notePath, day)));
33
- const at = new Set((await entries).filter(entry => entry.text.includes(hit.text)).map(entry => entry.at));
34
- return at.size === 1 ? [...at][0] : undefined;
35
- };
36
- }
37
- /** Every entry of a daily note, as readers get its text (its lines up to the next entry's), with the moment it was saved. */
38
- async function entriesOf(notePath, day) {
39
- let stored;
40
- try {
41
- stored = await fs_1.default.promises.readFile(notePath, 'utf8');
42
- }
43
- catch {
44
- return []; // gone since it was indexed: the hit keeps its note's date
45
- }
46
- // An entry starts where the stored text says one does (dailyEntryStarts), and its words are the decoded lines, which
47
- // the decode changes in place, never adding or removing one.
48
- const storedLines = stored.split('\n');
49
- const lines = (0, dailyNote_1.decodeDailyNoteText)(stored).split('\n');
50
- const starts = storedLines.flatMap((line, i) => ((0, dailyNote_1.parseEntryHeader)(line) ? [i] : []));
51
- return starts.map((start, k) => ({
52
- text: lines.slice(start, starts[k + 1] ?? lines.length).join('\n'),
53
- at: (0, dailyNote_1.entryInstant)(day, (0, dailyNote_1.parseEntryHeader)(storedLines[start])),
54
- }));
55
- }
@@ -1,162 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.entityScopeFilter = entityScopeFilter;
4
- exports.liveEntityFilter = liveEntityFilter;
5
- exports.lookupEntity = lookupEntity;
6
- const readCache_1 = require("../db/readCache");
7
- const statements_1 = require("../db/statements");
8
- const scope_1 = require("../mvp/scope");
9
- /** The note that owns the lifecycle entities, the subjects remembered facts are about (mirrors lifecycle.ts). */
10
- const LIFECYCLE_NOTE_PATH = '_mcp_';
11
- /**
12
- * The entities that belong to `scope`, as a filter for `lookupEntity` and for counting entities. A lifecycle subject
13
- * belongs to every scope one of its facts was saved in (under any spelling of it: savedScopeSqlPredicate), current or
14
- * superseded, the rule retrieve_memory judges a subject's existence by: its source note is the reserved name `_mcp_`,
15
- * not a path, and says nothing about scope (read as a path it would be global, so a global lookup would name subjects
16
- * saved only in a named scope, and a named scope would find none of its own). Every other entity belongs to the scope
17
- * its source note's path names.
18
- */
19
- function entityScopeFilter(scope) {
20
- const saved = (0, scope_1.savedScopeSqlPredicate)(scope, 'f.scope');
21
- const path = (0, scope_1.scopeSqlPredicate)(scope, 'entities.source_note');
22
- return {
23
- sql: `CASE WHEN entities.source_note = ? THEN EXISTS (SELECT 1 FROM facts f WHERE f.subject_entity_id = entities.id AND ${saved.sql}) ELSE ${path.sql} END`,
24
- params: [LIFECYCLE_NOTE_PATH, ...saved.params, ...path.params],
25
- };
26
- }
27
- /**
28
- * The entities of every scope, as a filter for `lookupEntity` and for counting entities: a lifecycle subject only while
29
- * it has a fact that is not retired (memory/lifecycle.ts moves a retired fact out of `facts`), the rule
30
- * entityScopeFilter applies within one scope; every other entity.
31
- */
32
- function liveEntityFilter() {
33
- return {
34
- sql: `CASE WHEN entities.source_note = ? THEN EXISTS (SELECT 1 FROM facts f WHERE f.subject_entity_id = entities.id) ELSE 1 END`,
35
- params: [LIFECYCLE_NOTE_PATH],
36
- };
37
- }
38
- /**
39
- * Of entities that match a name equally well, which one answers: a remembered fact's subject first, then by source
40
- * note path, then the oldest. Left to SQLite, the first row read answers, which is the note indexing happened to
41
- * write first, and concurrent indexing writes notes in no fixed order.
42
- */
43
- function tieOrder(table = '') {
44
- return `(${table}source_note = '${LIFECYCLE_NOTE_PATH}') DESC, ${table}source_note, ${table}id`;
45
- }
46
- /**
47
- * The units that occur in `column` of any entity, folded the way SQLite's `lower` folds (A to Z only: a build with ICU would
48
- * fold more, and `lowerIsAscii` says whether this one does). Null when a value is not text.
49
- */
50
- function unitsOf(db, column) {
51
- const units = new Uint32Array(0x10000 / 32);
52
- for (const value of (0, statements_1.prepareOnce)(db, `SELECT ${column} FROM entities`).pluck().all()) {
53
- if (value === null)
54
- continue; // instr(NULL, x) is NULL: such a row never matches
55
- if (typeof value !== 'string')
56
- return null;
57
- for (let i = 0; i < value.length; i++) {
58
- const unit = value.charCodeAt(i);
59
- const folded = unit >= 65 && unit <= 90 ? unit + 32 : unit;
60
- units[folded >> 5] |= 1 << (folded & 31);
61
- }
62
- }
63
- return units;
64
- }
65
- /** Whether `lower` leaves everything but A to Z alone, which is what `unitsOf` assumes. */
66
- function lowerIsAscii(db) {
67
- const probe = 'ÀÉÎÕÜÇΩДЖİß';
68
- return (0, statements_1.prepareOnce)(db, 'SELECT lower(?)').pluck().get(`${probe}ABC`) === `${probe}abc`;
69
- }
70
- /**
71
- * The longest canonical name, alias and source path, in SQLite characters (one code point, not a JavaScript UTF-16
72
- * unit), and the units that occur in the names and in the paths. Read once until the next write. An alias list that cannot
73
- * be read counts as unbounded, so that query still runs.
74
- */
75
- function entityNameBounds(db) {
76
- return (0, readCache_1.readUntilWrite)(db, 'entityLookupBounds', () => {
77
- const row = (0, statements_1.prepareOnce)(db, `SELECT COALESCE(MAX(length(canonical_name)), 0) AS name, COALESCE(MAX(length(source_note)), 0) AS note FROM entities`).get();
78
- let alias = Number.POSITIVE_INFINITY;
79
- try {
80
- alias = (0, statements_1.prepareOnce)(db, `SELECT COALESCE(MAX(length(alias.value)), 0) AS n FROM entities AS e, json_each(e.aliases_json) AS alias`).pluck().get();
81
- }
82
- catch { /* malformed aliases: do not skip the alias query */ }
83
- const folded = lowerIsAscii(db);
84
- return {
85
- name: Number(row.name),
86
- alias: Number(alias),
87
- note: Number(row.note),
88
- names: folded ? unitsOf(db, 'canonical_name') : null,
89
- notes: folded ? unitsOf(db, 'source_note') : null,
90
- };
91
- });
92
- }
93
- /**
94
- * False when `name` cannot be a substring of any value whose units are `units`: it has a unit (capitals folded) that none
95
- * of them has. A name with a half of a pair, or a NUL, is left to SQLite.
96
- */
97
- function mayBeWithin(units, name) {
98
- if (units === null)
99
- return true;
100
- for (let i = 0; i < name.length; i++) {
101
- const unit = name.charCodeAt(i);
102
- if (unit === 0 || (unit >= 0xd800 && unit <= 0xdfff))
103
- return true;
104
- const folded = unit >= 65 && unit <= 90 ? unit + 32 : unit;
105
- if ((units[folded >> 5] & (1 << (folded & 31))) === 0)
106
- return false;
107
- }
108
- return true;
109
- }
110
- /**
111
- * Resolve an entity by name or alias, then by substring of a canonical name,
112
- * then by substring of its source note path.
113
- *
114
- * `filter`, when given, is applied inside every candidate query — the scope has
115
- * to constrain the search, not the answer, or the search still reads the rows
116
- * the caller may not see.
117
- */
118
- function lookupEntity(db, name, filter) {
119
- const where = filter ? ` AND (${filter.sql})` : '';
120
- const params = filter ? filter.params : [];
121
- // `lower` and NOCASE keep a string's length, and `instr` cannot match a needle longer than the haystack. The length
122
- // is SQLite's: a JavaScript `.length` counts one code point above U+FFFF as two and would skip a real name.
123
- const bounds = entityNameBounds(db);
124
- const qlen = (0, statements_1.prepareOnce)(db, 'SELECT length(?) AS n').get(name).n;
125
- const fits = (max) => qlen === null || qlen <= max;
126
- if (!fits(bounds.name) && !fits(bounds.alias) && !fits(bounds.note))
127
- return null;
128
- if (fits(bounds.name)) {
129
- const byName = (0, statements_1.prepareOnce)(db, `SELECT * FROM entities WHERE canonical_name = ? COLLATE NOCASE${where} ORDER BY ${tieOrder()} LIMIT 1`)
130
- .get(name, ...params);
131
- if (byName)
132
- return toRow(byName);
133
- }
134
- // Unaliased, as in the other three queries, so a filter can name the row's own columns as `entities.<column>`.
135
- if (fits(bounds.alias)) {
136
- const byAlias = (0, statements_1.prepareOnce)(db, `SELECT entities.* FROM entities, json_each(entities.aliases_json) AS alias
137
- WHERE lower(alias.value) = lower(?)${where}
138
- ORDER BY ${tieOrder('entities.')}
139
- LIMIT 1`)
140
- .get(name, ...params);
141
- if (byAlias)
142
- return toRow(byAlias);
143
- }
144
- // Partial match: canonical_name contains the search term (e.g. "User" → "USER.md — About User")
145
- if (fits(bounds.name) && mayBeWithin(bounds.names, name)) {
146
- const byPartial = (0, statements_1.prepareOnce)(db, `SELECT * FROM entities WHERE instr(lower(canonical_name), lower(?)) > 0${where} ORDER BY length(canonical_name) ASC, ${tieOrder()} LIMIT 1`)
147
- .get(name, ...params);
148
- if (byPartial)
149
- return toRow(byPartial);
150
- }
151
- // Path match: source note path contains the search term (e.g. "Cora" → notes in .../Cora/...)
152
- if (fits(bounds.note) && mayBeWithin(bounds.notes, name)) {
153
- const byPath = (0, statements_1.prepareOnce)(db, `SELECT * FROM entities WHERE instr(lower(source_note), lower(?)) > 0${where} ORDER BY length(canonical_name) ASC, ${tieOrder()} LIMIT 1`)
154
- .get(name, ...params);
155
- if (byPath)
156
- return toRow(byPath);
157
- }
158
- return null;
159
- }
160
- function toRow(r) {
161
- return { id: r.id, type: r.type, canonicalName: r.canonical_name, aliases: JSON.parse(r.aliases_json ?? '[]'), sourceNote: r.source_note, confidence: r.confidence };
162
- }