renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,422 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.READER_SESSION_CAP = exports.LAYOUT_MIN_TURNS = void 0;
4
- exports.readerLayoutEnabled = readerLayoutEnabled;
5
- exports.readerLayoutSkipsAdvice = readerLayoutSkipsAdvice;
6
- exports.readerLayoutEnabledFor = readerLayoutEnabledFor;
7
- exports.readerTimelineEnabled = readerTimelineEnabled;
8
- exports.isCurrentValueQuestion = isCurrentValueQuestion;
9
- exports.currentValueTimeline = currentValueTimeline;
10
- exports.isConversation = isConversation;
11
- exports.conversationNoteMatcher = conversationNoteMatcher;
12
- exports.readerSessionCap = readerSessionCap;
13
- exports.readerBlockOrder = readerBlockOrder;
14
- exports.isHeaderOnly = isHeaderOnly;
15
- exports.layoutHitsForReader = layoutHitsForReader;
16
- exports.layoutPage = layoutPage;
17
- exports.readerLayoutPage = readerLayoutPage;
18
- exports.readerLayoutOf = readerLayoutOf;
19
- const relativeTimeHints_1 = require("./relativeTimeHints");
20
- const orchestrator_1 = require("./orchestrator");
21
- const questionSpan_1 = require("./questionSpan");
22
- const readerEvents_1 = require("./readerEvents");
23
- const conversationShape_1 = require("./conversationShape");
24
- const adviceQuestion_1 = require("./adviceQuestion");
25
- /**
26
- * How a search reply is laid out for the model that reads it. Retrieval picks the window and orders it; this only
27
- * changes how what it picked is handed over, and it never cuts or rewrites a word of an excerpt it shows:
28
- *
29
- * - one hit per session: the chunks of a note on one date, which the newest-first layout left apart (a session's header,
30
- * which carries its date, could come after the turns it dates, and turns of other sessions in between), come as one hit,
31
- * in the note's own order, so every turn sits under the date of its session;
32
- * - a session whose only chunk is its header (`# Session x` and a `Timestamp:` line) is no hit: it dated nothing the reader
33
- * can see and took a place in the reply (a third of the hits of the 2026-10-16 LongMemEval-S run);
34
- * - the blocks stand where the page had them (newest first by default). RENOR_READER_ORDER=relevance puts them best match
35
- * first instead (a block's score is its best chunk's, ties newest first) and the reply says so (`rankedBy: score`,
36
- * readerLayoutPage; only a page whose ordinary notes are all conversations). It is off by default: on RenorBench v2 it puts a
37
- * superseded value ahead of the current one (tests/eval/renorbench-v2.test.ts);
38
- * - RENOR_READER_SESSION_CAP=N lets a session show at most N chunks besides its header, except that every chunk the query
39
- * matched stays: the cap only stops the statements the session expansion added. Off by default (0): on RenorBench v2 a cap
40
- * of 10 drops the remark a question asks about, which only the expansion delivers;
41
- * - a session that reaches the reader without its header (the date line is its first chunk) opens with
42
- * `[Session date: <date>]`;
43
- * - each time phrase that names one point in time ("three weeks ago", "last Thursday") is resolved to its date below
44
- * the excerpt (relativeTimeHints.ts).
45
- *
46
- * Whole chunks leave (a header stub, a statement over the cap); no chunk is cut or reworded.
47
- *
48
- * Only notes shaped like a conversation (conversationShape.ts) are laid out: a session is one conversation, and merging
49
- * the chunks of a document or the entries of a daily note would break "a hit is one memory".
50
- *
51
- * Measured on LongMemEval-S dev100 with `gemma4:31b` as reader and judge: see docs/evals/results/READER-LAYOUT-20261014.md.
52
- * Not applied to get_context (its budget cuts whole items by relevance) or to any hit that is no chunk of an ordinary
53
- * note: a remembered fact, a ledger event, a correction, the evidence or digest table. Kill switch: RENOR_READER_LAYOUT=0.
54
- *
55
- * On by default, and only on top of this layout (`RENOR_READER_EVENTS=0` switches it off; then a layout-on page is
56
- * byte-identical): when the question asks for the order or the date of something the user did, one line per matching
57
- * statement of the user's, dated and oldest first, is placed above the excerpts (readerEvents.ts). A duration question and
58
- * any other question are left untouched.
59
- *
60
- * Opt-in, and only on top of this layout (`RENOR_READER_TIMELINE=1`; default off, so a layout-on page is byte-identical):
61
- * when the question asks what a value is now, one timeline of the user turns that state it is placed above the excerpts.
62
- * Oldest first, the latest line marked, earlier lines kept. It quotes those turns and does not cut or reword an excerpt,
63
- * invent a value, or do arithmetic on a session header (a header date orders two different calendar days only; a shared
64
- * day is not a clock). A duration question or an order question is left untouched.
65
- */
66
- function readerLayoutEnabled() {
67
- const v = (process.env.RENOR_READER_LAYOUT ?? '').trim().toLowerCase();
68
- return !(v === '0' || v === 'false' || v === 'no' || v === 'off');
69
- }
70
- /**
71
- * Default off. `1`, `true`, `yes` or `on` leaves a query that asks to be advised (adviceQuestion.ts: a recommendation, a
72
- * suggestion, tips, ideas, an opinion) without the layout, so its page is the one RENOR_READER_LAYOUT=0 gives. Every other query,
73
- * and every query with this unset, is laid out exactly as before.
74
- */
75
- function readerLayoutSkipsAdvice() {
76
- const v = (process.env.RENOR_READER_LAYOUT_SKIP_ADVICE ?? '').trim().toLowerCase();
77
- return v === '1' || v === 'true' || v === 'yes' || v === 'on';
78
- }
79
- /** Is the page of this query laid out? The kill switch decides first, then the advice exception. */
80
- function readerLayoutEnabledFor(query) {
81
- return readerLayoutEnabled() && !(readerLayoutSkipsAdvice() && (0, adviceQuestion_1.asksForAdvice)(query));
82
- }
83
- /** Default off. `1` / `true` / `yes` / `on` places a current-value timeline above a layout-on page. */
84
- function readerTimelineEnabled(env = process.env) {
85
- const v = (env.RENOR_READER_TIMELINE ?? '').trim().toLowerCase();
86
- return v === '1' || v === 'true' || v === 'yes' || v === 'on';
87
- }
88
- // A duration or an order stays byte-identical. "Ago" and "last Tuesday" are a point in the past, not a changed value.
89
- const DURATION_QUESTION = /\b(?:how (?:many|much) (?:days|weeks|months|years)|how long|how much time|how old)\b|\b(?:days|weeks|months|years)\b[^?]{0,24}\b(?:ago|passed)\b|\bago\b|\bpassed between\b|\blast (?:week|month|year|weekend|monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b/i;
90
- const ORDER_QUESTION = /\border\b|\b(?:first to last|earliest to latest|happened first|came first)\b|\b(?:which|who)\b[^?]{0,80}\bfirst\b/i;
91
- const CURRENT_VALUE_QUESTION = /\b(?:now|currently|current|most recent(?:ly)?|latest|so far|previous(?:ly)?|updated|initially|what changed|used to|how many|how often|how much|what|where|did i|do i|am i|is my|have i)\b/i;
92
- /** A knowledge-update-shaped question: what the value is now, or what it was before it changed. Not a duration or an order. */
93
- function isCurrentValueQuestion(query) {
94
- const q = (query.split('\n').pop() ?? query).trim();
95
- if (q === '' || DURATION_QUESTION.test(q) || ORDER_QUESTION.test(q))
96
- return false;
97
- return CURRENT_VALUE_QUESTION.test(q);
98
- }
99
- const QUOTE_STOP = new Set([
100
- 'what', 'where', 'when', 'which', 'who', 'whom', 'how', 'many', 'much', 'often', 'does', 'did', 'do', 'have', 'has',
101
- 'had', 'was', 'were', 'been', 'being', 'this', 'that', 'these', 'those', 'with', 'from', 'your', 'about', 'into',
102
- 'over', 'under', 'after', 'before', 'between', 'since', 'until', 'while', 'just', 'still', 'also', 'than', 'then',
103
- 'them', 'they', 'their', 'there', 'here', 'would', 'could', 'should', 'will', 'shall', 'can', 'may', 'might', 'must',
104
- 'some', 'any', 'more', 'most', 'other', 'only', 'same', 'very', 'really', 'around', 'time', 'times', 'when',
105
- ]);
106
- function contentTokens(question) {
107
- const words = question.toLowerCase().match(/[a-z0-9$][a-z0-9$'-]{3,}/g) ?? [];
108
- return new Set(words.filter(word => !QUOTE_STOP.has(word) && !/^\d+$/.test(word)));
109
- }
110
- function sameWord(a, b) {
111
- if (a === b)
112
- return true;
113
- const [longer, shorter] = a.length >= b.length ? [a, b] : [b, a];
114
- return longer === `${shorter}s` || longer === `${shorter}es`;
115
- }
116
- function sharesToken(sentence, tokens) {
117
- const words = sentence.toLowerCase().match(/[a-z0-9$][a-z0-9$'-]{3,}/g) ?? [];
118
- return words.some(word => {
119
- for (const token of tokens)
120
- if (sameWord(word, token))
121
- return true;
122
- return false;
123
- });
124
- }
125
- function headingOf(line) {
126
- const match = /^#{1,6}\s+(.*)$/.exec(line.trim());
127
- return match ? match[1].trim() : null;
128
- }
129
- function headingWords(heading) {
130
- return heading.toLowerCase().replace(/[\p{N}\p{P}\p{S}\p{Z}]+/gu, ' ').trim().split(' ').filter(Boolean);
131
- }
132
- function isUserTurnHeading(line) {
133
- const heading = headingOf(line);
134
- if (heading === null || !(0, conversationShape_1.isTurnLabel)(heading))
135
- return false;
136
- const words = headingWords(heading);
137
- return words.includes('user') || words.includes('human');
138
- }
139
- function isTurnHeading(line) {
140
- const heading = headingOf(line);
141
- return heading !== null && (0, conversationShape_1.isTurnLabel)(heading);
142
- }
143
- /** The user's own words in one laid-out hit, one string per user turn. Assistant turns are not quoted. */
144
- function userTurnBodies(text) {
145
- const lines = text.split(/\r?\n/);
146
- const bodies = [];
147
- for (let i = 0; i < lines.length; i++) {
148
- if (!isUserTurnHeading(lines[i]))
149
- continue;
150
- const buf = [];
151
- for (let j = i + 1; j < lines.length; j++) {
152
- const trimmed = lines[j].trim();
153
- if (isTurnHeading(lines[j]))
154
- break;
155
- if (trimmed.startsWith('[Counted from this session\'s date') || trimmed.startsWith('[Session date:'))
156
- continue;
157
- buf.push(lines[j]);
158
- }
159
- const body = buf.join(' ').replace(/\s+/g, ' ').trim();
160
- if (body !== '')
161
- bodies.push(body);
162
- }
163
- return bodies;
164
- }
165
- function quoteSentences(body, tokens) {
166
- const sentences = body.split(/(?<=[.!?])\s+/).map(sentence => sentence.trim()).filter(sentence => sentence !== '');
167
- const kept = (sentences.length > 0 ? sentences : [body]).filter(sentence => sharesToken(sentence, tokens));
168
- if (kept.length === 0)
169
- return null;
170
- const best = kept.slice().sort((a, b) => Number(sharesToken(b, tokens)) - Number(sharesToken(a, tokens)) || b.length - a.length)[0];
171
- const text = kept.length === 1 ? kept[0] : [best, ...kept.filter(sentence => sentence !== best)].slice(0, 2).join(' ');
172
- if (text.length <= 500)
173
- return text;
174
- const lower = best.toLowerCase();
175
- let at = -1;
176
- for (const token of tokens) {
177
- const found = lower.indexOf(token);
178
- if (found >= 0 && (at < 0 || found < at))
179
- at = found;
180
- }
181
- if (at < 0)
182
- return best.slice(0, 500).trim();
183
- const start = Math.max(0, at - 180);
184
- return best.slice(start, Math.min(best.length, start + 500)).trim();
185
- }
186
- /** Calendar day only. The clock on a shared day is not an order: those stamps are often the question's own day. */
187
- function calendarDay(sessionDate) {
188
- const match = /^(\d{4})[/-](\d{2})[/-](\d{2})/.exec(sessionDate ?? '');
189
- return match ? `${match[1]}-${match[2]}-${match[3]}` : null;
190
- }
191
- function sessionIndex(notePath) {
192
- const base = notePath.split('/').pop() ?? notePath;
193
- const match = /_(\d+)(?:-|\.|$)/.exec(base);
194
- return match ? Number(match[1]) : null;
195
- }
196
- /**
197
- * The timeline line, or null when this question is not a current-value question or no user turn states one.
198
- * Nothing here is invented: every quote is a substring of a user turn that is already in `hits`.
199
- */
200
- function currentValueTimeline(hits, query) {
201
- if (!isCurrentValueQuestion(query))
202
- return null;
203
- const tokens = contentTokens(query.split('\n').pop() ?? query);
204
- if (tokens.size === 0)
205
- return null;
206
- const stated = [];
207
- const seen = new Set();
208
- hits.forEach((hit, appearance) => {
209
- const day = calendarDay(hit.session_date);
210
- const index = sessionIndex(hit.notePath);
211
- for (const body of userTurnBodies(hit.text)) {
212
- const text = quoteSentences(body, tokens);
213
- if (text === null)
214
- continue;
215
- // One statement per note: the same words in a later session are a later statement (A, B, A ends on A), not a repeat.
216
- const key = `${hit.notePath}\u0000${text.toLowerCase()}`;
217
- if (seen.has(key))
218
- continue;
219
- seen.add(key);
220
- stated.push({ day, index, chunk: hit.chunkId ?? 0, appearance, text });
221
- }
222
- });
223
- if (stated.length === 0)
224
- return null;
225
- stated.sort((a, b) => {
226
- if (a.day !== null && b.day !== null && a.day !== b.day)
227
- return a.day < b.day ? -1 : 1;
228
- if (a.index !== null && b.index !== null && a.index !== b.index)
229
- return a.index - b.index;
230
- if (a.chunk !== b.chunk)
231
- return a.chunk - b.chunk;
232
- return a.appearance - b.appearance;
233
- });
234
- const lines = stated.map((value, i) => (i === stated.length - 1 ? `- latest: "${value.text}"` : `- "${value.text}"`));
235
- return ['Stated by the user, oldest first. Earlier values stay. The last line is the latest.', ...lines].join('\n');
236
- }
237
- function withCurrentValueTimeline(hits, query) {
238
- if (!readerTimelineEnabled() || query === undefined || query.trim() === '')
239
- return hits;
240
- const text = currentValueTimeline(hits, query);
241
- if (text === null)
242
- return hits;
243
- return [{ notePath: '', text, score: 0, reason: 'timeline' }, ...hits];
244
- }
245
- /**
246
- * A note of turns: at least `LAYOUT_MIN_TURNS` of its distinct heading paths end in a speaker or turn label (conversationShape.ts
247
- * isTurnLabel) and those are at least half of its paths. One exchange, a question and its answer, is a conversation here: the
248
- * session expansion's own rule asks for four turns because it cuts statements, and this one only joins what a session already is.
249
- */
250
- exports.LAYOUT_MIN_TURNS = 2;
251
- function isConversation(headingPaths) {
252
- const distinct = new Set(headingPaths);
253
- let turns = 0;
254
- for (const headingPath of distinct)
255
- if ((0, conversationShape_1.isTurnLabel)(headingPath.split(' > ').pop() ?? ''))
256
- turns += 1;
257
- return turns >= exports.LAYOUT_MIN_TURNS && turns * 2 >= distinct.size;
258
- }
259
- /** Is this stored note path a conversation? Told from the headings of its chunks, once per note. */
260
- function conversationNoteMatcher(db) {
261
- const select = db.prepare('SELECT heading_path AS headingPath FROM chunks WHERE note_path = ?');
262
- const known = new Map();
263
- return storedNotePath => {
264
- let shaped = known.get(storedNotePath);
265
- if (shaped === undefined) {
266
- shaped = isConversation(select.all(storedNotePath).map(row => row.headingPath));
267
- known.set(storedNotePath, shaped);
268
- }
269
- return shaped;
270
- };
271
- }
272
- function isNoteChunk(hit, isLedger) {
273
- return (hit.chunkId ?? 0) > 0 && hit.notePath !== '' && !isLedger(hit.notePath) && hit.author === undefined && hit.replaced === undefined;
274
- }
275
- /** The most chunks of one session's block (its header aside); RENOR_READER_SESSION_CAP, 0 (the default) is no cap. */
276
- exports.READER_SESSION_CAP = 0;
277
- function readerSessionCap(env = process.env) {
278
- const raw = (env.RENOR_READER_SESSION_CAP ?? '').trim();
279
- if (raw === '')
280
- return exports.READER_SESSION_CAP;
281
- const n = Number(raw);
282
- return Number.isInteger(n) && n >= 0 ? n : exports.READER_SESSION_CAP;
283
- }
284
- /** Blocks keep the page's own (newest-first) order; RENOR_READER_ORDER=relevance puts them best match first. */
285
- function readerBlockOrder(env = process.env) {
286
- return (env.RENOR_READER_ORDER ?? '').trim().toLowerCase() === 'relevance' ? 'relevance' : 'date';
287
- }
288
- /**
289
- * A chunk that is only a session's header: headings and `Timestamp:` lines, no turn. It dates the turns under it and says
290
- * nothing itself; on its own it is a stub that takes a place in the reply (a third of the hits of the 2026-10-16 run).
291
- */
292
- function isHeaderOnly(text) {
293
- const lines = text.split(/\r?\n/).map(line => line.trim()).filter(line => line !== '' && !/^#{1,6}\s/.test(line));
294
- return lines.length > 0 && lines.every(line => /^Timestamp:/i.test(line));
295
- }
296
- /** A hit of an ordinary note (a chunk the layout could place, a daily entry, a document paragraph): not a fact, a ledger event, a correction or the evidence and digest tables (`_renor/…`). */
297
- function isOrdinaryNoteHit(hit, isLedger) {
298
- return hit.notePath !== '' && !hit.notePath.startsWith('_renor/') && !isLedger(hit.notePath) && hit.author === undefined && hit.replaced === undefined;
299
- }
300
- function layoutHitsForReader(hits, opts) {
301
- return layoutPage(hits, opts).hits;
302
- }
303
- /**
304
- * `byRelevance` is true when the blocks were placed best match first, which the reply then says (`rankedBy: score`). It is
305
- * done only to a page whose every chunk of an ordinary note is a conversation's: a daily note's entries or a document's
306
- * paragraphs on the same page keep the order the page gave them, and a page that is half one order and half the other must
307
- * not claim either key.
308
- */
309
- function layoutPage(hits, opts) {
310
- const chunk = (hit) => isNoteChunk(hit, opts.isLedger) && (opts.isConversation?.(hit.notePath) ?? true);
311
- const cap = opts.sessionCap ?? readerSessionCap();
312
- const order = opts.order ?? readerBlockOrder();
313
- const laid = hits.map(hit => (chunk(hit) ? { ...hit, text: (0, relativeTimeHints_1.withRelativeTimeHints)(hit.text, hit.session_date) } : hit));
314
- // One block per conversation on one date: every chunk of it under that date, wherever the page had put it.
315
- const keyOf = (hit) => (chunk(hit) ? `${hit.notePath}\0${hit.session_date ?? ''}` : null);
316
- const blocks = new Map();
317
- laid.forEach((hit, index) => {
318
- const key = keyOf(hit);
319
- if (key === null)
320
- return;
321
- const block = blocks.get(key) ?? { header: [], body: [], first: index };
322
- (isHeaderOnly(hit.text) ? block.header : block.body).push(hit);
323
- blocks.set(key, block);
324
- });
325
- const made = new Map();
326
- for (const [key, block] of blocks)
327
- made.set(key, blockHit(block, cap));
328
- // The blocks stand in the places the page gave its chunks; anything else keeps its place.
329
- const slots = new Set(laid.flatMap((hit, index) => (keyOf(hit) !== null && blocks.get(keyOf(hit)).first === index ? [index] : [])));
330
- let placed = [...slots].map(index => keyOf(laid[index])).filter(key => made.get(key) !== null);
331
- const byRelevance = order === 'relevance' && placed.length > 0 && hits.every(hit => !isOrdinaryNoteHit(hit, opts.isLedger) || chunk(hit));
332
- if (byRelevance) {
333
- const at = new Map(placed.map((key, i) => [key, i]));
334
- placed = [...placed].sort((a, b) => made.get(b).score - made.get(a).score
335
- || ((0, orchestrator_1.sessionDateMs)(made.get(b).session_date) || 0) - ((0, orchestrator_1.sessionDateMs)(made.get(a).session_date) || 0)
336
- || at.get(a) - at.get(b));
337
- }
338
- const out = [];
339
- let next = 0;
340
- laid.forEach((hit, index) => {
341
- const key = keyOf(hit);
342
- if (key === null)
343
- out.push(hit);
344
- else if (slots.has(index) && made.get(key) !== null)
345
- out.push(made.get(placed[next++]));
346
- });
347
- const dated = out.map(hit => (chunk(hit) && hit.session_date && !/Timestamp:/i.test(hit.text)
348
- ? { ...hit, text: `[Session date: ${hit.session_date}]\n${hit.text}` }
349
- : hit));
350
- // RENOR_READER_DURATION (default on; `0` switches it off): one line of dates the user wrote, first, for a "how many days between" question (questionSpan.ts).
351
- // It reads the chunks as they were found: joined, a turn's continuation would sit under the speaker before it.
352
- // RENOR_READER_EVENTS (default on; `0` switches it off) adds event lines for an order or a date question; its gate leaves a duration question alone, so at most one of the two lines is added.
353
- // RENOR_READER_TIMELINE (default off): a current-value timeline for a "what is it now" question; a duration or an order question is left alone.
354
- // Each such line comes first, outside the order rankedBy names, as the evidence hit does. The event lines too read the
355
- // conversations as the page found them: a session that came as its header alone is no hit, but its turns are still read.
356
- return {
357
- hits: withCurrentValueTimeline(withEventLines((0, questionSpan_1.withQuestionSpan)(dated, opts.query, { source: hits }), hits.filter(chunk), opts), opts.query),
358
- byRelevance,
359
- };
360
- }
361
- /** The event lines, as the first hit (reason "events"), when the flag is on and the question asks for an order or a date. `sources` are the conversation chunks they read. */
362
- function withEventLines(hits, sources, opts) {
363
- if (!(0, readerEvents_1.readerEventsEnabled)() || opts.query === undefined || opts.turnsOf === undefined || opts.query.trim() === '')
364
- return hits;
365
- const text = (0, readerEvents_1.eventLines)(sources, opts.query, opts.turnsOf, { scoreTurns: opts.scoreTurns });
366
- if (text === null)
367
- return hits;
368
- return [{ notePath: '', text, score: 0, reason: 'events' }, ...hits];
369
- }
370
- /**
371
- * One conversation's hit: its header first when it came, then its turns in the conversation's own order, with the best
372
- * score and reason of its chunks. A session that came as a header and nothing else is no hit. Over the cap, every chunk the
373
- * query matched stays and the statements the session expansion added are taken in the order they came, as many as the cap leaves room for.
374
- */
375
- function blockHit(block, cap) {
376
- if (block.body.length === 0)
377
- return null;
378
- const byPlace = (a, b) => (a.chunkId ?? 0) - (b.chunkId ?? 0);
379
- let body = block.body;
380
- if (cap > 0 && body.length > cap) {
381
- // What the query matched always stays; the cap only limits the statements the session expansion added.
382
- const expanded = body.filter(h => h.reason === orchestrator_1.SESSION_EXPAND_REASON);
383
- const room = Math.max(0, cap - (body.length - expanded.length));
384
- const kept = new Set([...body.filter(h => h.reason !== orchestrator_1.SESSION_EXPAND_REASON), ...expanded.slice(0, room)]);
385
- body = body.filter(h => kept.has(h));
386
- }
387
- const run = [...block.header, ...body].sort(byPlace);
388
- if (run.length === 1)
389
- return run[0];
390
- const best = run.reduce((a, b) => (b.score > a.score ? b : a));
391
- const similarities = run.flatMap(h => (typeof h.similarity === 'number' ? [h.similarity] : []));
392
- return {
393
- ...run[0],
394
- text: run.map(h => h.text).join('\n\n'),
395
- score: best.score,
396
- reason: best.reason,
397
- ...(similarities.length > 0 ? { similarity: Math.max(...similarities) } : {}),
398
- };
399
- }
400
- /**
401
- * The page search_memory shows, laid out for its reader when RENOR_READER_LAYOUT is on (the chunk ids the layout needed
402
- * leave with it); otherwise the page as it was. `rankedBy` is the key the returned page is in order of. `enabled` is the
403
- * answer for this query (readerLayoutEnabledFor).
404
- */
405
- function readerLayoutPage(rankedBy, hits, isLedger, events = {}, enabled = readerLayoutEnabled()) {
406
- if (!enabled)
407
- return { hits, rankedBy };
408
- // A bare string is the question alone (the duration line's form); the object also carries what the event lines read.
409
- const input = typeof events === 'string' ? { query: events } : events;
410
- const laid = layoutPage(hits, { rankedBy, isLedger, ...input });
411
- return {
412
- hits: laid.hits.map(hit => {
413
- const { chunkId: _chunkId, ...rest } = hit;
414
- return rest;
415
- }),
416
- rankedBy: laid.byRelevance ? 'score' : rankedBy,
417
- };
418
- }
419
- /** The hits of readerLayoutPage alone. */
420
- function readerLayoutOf(rankedBy, hits, isLedger, events = {}, enabled = readerLayoutEnabled()) {
421
- return readerLayoutPage(rankedBy, hits, isLedger, events, enabled).hits;
422
- }
@@ -1,113 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.formatDay = formatDay;
4
- exports.relativeTimePoints = relativeTimePoints;
5
- exports.relativeTimeHints = relativeTimeHints;
6
- exports.withRelativeTimeHints = withRelativeTimeHints;
7
- const orchestrator_1 = require("./orchestrator");
8
- /**
9
- * Says, below an excerpt, which calendar date each relative time phrase in it points to, counted from the excerpt's own
10
- * session date: "three weeks ago" in a session of 2023/05/28 is 2023/05/07. A reader that has to do that arithmetic,
11
- * then the arithmetic of the question on top, often answers "I don't know" with both dates in front of it; the hint
12
- * does the first step and leaves the stored text, and the question's own arithmetic, alone.
13
- *
14
- * English only, and only phrases that name one point in time (a count of days, weeks, months or years ago; yesterday,
15
- * today; last <weekday>, week, month or year). Nothing is guessed: a phrase this does not match gets no hint, and a hit
16
- * without a parseable date gets none.
17
- */
18
- const NUMBER_WORDS = {
19
- a: 1, an: 1, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10, eleven: 11, twelve: 12,
20
- };
21
- const WEEKDAYS = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'];
22
- const DAY_NAMES = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
23
- const DAY_MS = 86_400_000;
24
- const AGO = new RegExp(`\\b(${Object.keys(NUMBER_WORDS).join('|')}|\\d{1,3})\\s+(day|week|month|year)s?\\s+ago\\b`, 'gi');
25
- const SIMPLE = /\b(yesterday|today|last\s+(?:week|month|year|sunday|monday|tuesday|wednesday|thursday|friday|saturday))\b/gi;
26
- function pad(n) {
27
- return String(n).padStart(2, '0');
28
- }
29
- function addMonths(date, months) {
30
- const out = new Date(date.getTime());
31
- const day = out.getUTCDate();
32
- out.setUTCDate(1);
33
- out.setUTCMonth(out.getUTCMonth() + months);
34
- const last = new Date(Date.UTC(out.getUTCFullYear(), out.getUTCMonth() + 1, 0)).getUTCDate();
35
- out.setUTCDate(Math.min(day, last));
36
- return out;
37
- }
38
- /** A day as the hit's own date style writes it: `2023/05/07 (Sun)` for the benchmark's slash dates, `2023-05-07` otherwise. */
39
- function formatDay(date, slashStyle) {
40
- const ymd = slashStyle ? `${date.getUTCFullYear()}/${pad(date.getUTCMonth() + 1)}/${pad(date.getUTCDate())}` : `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}`;
41
- return slashStyle ? `${ymd} (${DAY_NAMES[date.getUTCDay()]})` : ymd;
42
- }
43
- /** The phrases of one excerpt that name a point in time, in the order they appear, each phrase once. */
44
- function relativeTimePoints(text, sessionDate) {
45
- const ms = (0, orchestrator_1.sessionDateMs)(sessionDate);
46
- if (!Number.isFinite(ms) || sessionDate === undefined)
47
- return [];
48
- const slashStyle = /^\d{4}\//.test(sessionDate);
49
- const base = new Date(Math.floor(ms / DAY_MS) * DAY_MS);
50
- const found = [];
51
- for (const m of text.matchAll(AGO)) {
52
- const raw = m[1].toLowerCase();
53
- const n = raw in NUMBER_WORDS ? NUMBER_WORDS[raw] : Number(raw);
54
- const unit = m[2].toLowerCase();
55
- const date = unit === 'day' ? new Date(base.getTime() - n * DAY_MS)
56
- : unit === 'week' ? new Date(base.getTime() - 7 * n * DAY_MS)
57
- : unit === 'month' ? addMonths(base, -n)
58
- : addMonths(base, -12 * n);
59
- found.push({ at: m.index ?? 0, phrase: m[0], date, value: formatDay(date, slashStyle), grain: 'day' });
60
- }
61
- for (const m of text.matchAll(SIMPLE)) {
62
- const phrase = m[1].toLowerCase().replace(/\s+/g, ' ');
63
- let date;
64
- let value;
65
- let grain = 'day';
66
- if (phrase === 'today') {
67
- date = base;
68
- value = formatDay(date, slashStyle);
69
- }
70
- else if (phrase === 'yesterday') {
71
- date = new Date(base.getTime() - DAY_MS);
72
- value = formatDay(date, slashStyle);
73
- }
74
- else if (phrase === 'last week') {
75
- date = new Date(base.getTime() - 7 * DAY_MS);
76
- value = formatDay(date, slashStyle);
77
- }
78
- else if (phrase === 'last month') {
79
- date = addMonths(base, -1);
80
- value = formatDay(date, slashStyle).slice(0, 7);
81
- grain = 'month';
82
- }
83
- else if (phrase === 'last year') {
84
- date = new Date(Date.UTC(base.getUTCFullYear() - 1, 0, 1));
85
- value = String(base.getUTCFullYear() - 1);
86
- grain = 'year';
87
- }
88
- else {
89
- const target = WEEKDAYS.indexOf(phrase.slice(5));
90
- const back = ((base.getUTCDay() - target + 7) % 7) || 7;
91
- date = new Date(base.getTime() - back * DAY_MS);
92
- value = formatDay(date, slashStyle);
93
- }
94
- found.push({ at: m.index ?? 0, phrase: m[1].replace(/\s+/g, ' '), date, value, grain });
95
- }
96
- const seen = new Set();
97
- return found.sort((a, b) => a.at - b.at).flatMap(f => {
98
- const key = f.phrase.toLowerCase();
99
- if (seen.has(key))
100
- return [];
101
- seen.add(key);
102
- return [f];
103
- });
104
- }
105
- /** The hints for one excerpt: `"three weeks ago" is 2023/05/07 (Sun)`, in the order the phrases appear, each phrase once. */
106
- function relativeTimeHints(text, sessionDate) {
107
- return relativeTimePoints(text, sessionDate).map(point => `"${point.phrase}" is ${point.value}`);
108
- }
109
- /** The excerpt with its hints on one line below it; the text itself is untouched. An excerpt with no hint is returned as it was. */
110
- function withRelativeTimeHints(text, sessionDate) {
111
- const hints = relativeTimeHints(text, sessionDate);
112
- return hints.length === 0 ? text : `${text}\n\n[Counted from this session's date, ${sessionDate}: ${hints.join('; ')}]`;
113
- }
@@ -1,67 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.byPlace = byPlace;
4
- exports.byScoreThenPlace = byScoreThenPlace;
5
- exports.semanticSearch = semanticSearch;
6
- const readCache_1 = require("../db/readCache");
7
- const statements_1 = require("../db/statements");
8
- /**
9
- * Note path, then chunk id: the order of chunks whose scores are equal. A note's chunks are written top to bottom (by
10
- * the indexer, and by the importer, which copies them in that order), so within one note the id is the chunk's place,
11
- * the same in every index of the same notes. Across notes ids are not: they follow the order indexing wrote the notes
12
- * in, which concurrent indexing does not fix.
13
- */
14
- function byPlace(a, b) {
15
- if (a.notePath !== b.notePath)
16
- return a.notePath < b.notePath ? -1 : 1;
17
- return a.chunkId - b.chunkId;
18
- }
19
- /** Highest score first; equal scores by place (`byPlace`). */
20
- function byScoreThenPlace(a, b) {
21
- return b.score - a.score || byPlace(a, b);
22
- }
23
- async function semanticSearch(db, vectorIndex, provider, query, topK, probe,
24
- /** Only the chunks it shows are candidates: the nearest `topK` of them, however many others are nearer. */
25
- scope) {
26
- // A fresh/empty memory has nothing to compare against. Avoid paying the
27
- // provider/network cost for a query embedding that cannot produce a hit.
28
- // Ask vec0's rowid shadow table, not the virtual table: `SELECT ... FROM chunk_embeddings` without a
29
- // MATCH walks every vector (milliseconds at 10,000 notes, growing with the index) to learn "not empty".
30
- // `chunk_embeddings_rowids` holds one row per stored vector.
31
- const hasVectors = (0, statements_1.prepareOnce)(db, 'SELECT 1 FROM chunk_embeddings_rowids LIMIT 1').get();
32
- if (!hasVectors || topK <= 0)
33
- return [];
34
- const queryEmbedding = provider.embedQuery
35
- ? await provider.embedQuery(query)
36
- : (await provider.embed([query]))[0];
37
- if (probe)
38
- probe.queryVector = queryEmbedding;
39
- // The chunks of the `k` nearest vectors (of those `among` names, when it is given), best first, and how many vectors
40
- // were read: more than `k` when the farthest of them is tied, and the cut is made by place, not by where vectors are
41
- // stored; fewer only when there are no more.
42
- const nearest = (k, among) => {
43
- const hits = vectorIndex.search(queryEmbedding, k, among);
44
- if (hits.length === 0)
45
- return { vectors: 0, chunks: [] };
46
- const scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
47
- const rows = (0, statements_1.prepareOnce)(db, `SELECT id, note_path, heading_path, text, start_line, end_line FROM chunks WHERE id IN (SELECT value FROM json_each(?))`).all(JSON.stringify(hits.map(h => h.chunkId)));
48
- return {
49
- vectors: hits.length,
50
- chunks: rows
51
- .map(r => ({ notePath: r.note_path, chunkId: r.id, headingPath: r.heading_path, text: r.text, score: scoreMap.get(r.id) ?? 0, startLine: r.start_line, endLine: r.end_line }))
52
- .sort(byScoreThenPlace),
53
- };
54
- };
55
- // The query is embedded: nothing below waits for anything, so the nearest chunks are read in one transaction (inReadSnapshot).
56
- return (0, readCache_1.inReadSnapshot)(db, () => {
57
- if (!scope)
58
- return nearest(topK).chunks.slice(0, topK);
59
- // Judged before the cut: when fewer than `topK` of the nearest chunks are shown (other chunks are nearer), the nearest
60
- // of the shown chunks themselves are read instead, so however many others are nearer, none takes their places.
61
- const first = nearest(topK);
62
- const shown = first.chunks.filter(scope.shows);
63
- if (shown.length >= topK || first.vectors < topK)
64
- return shown.slice(0, topK);
65
- return nearest(topK, scope.chunkIds()).chunks.slice(0, topK);
66
- });
67
- }