renor-engine 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -661
- package/NOTICE +8 -6
- package/README.md +6 -6
- package/dist/.tsbuildinfo +1 -0
- package/dist/BUILD.json +2 -2
- package/dist/api/server.js +10 -8
- package/dist/buildIdentity.js +18 -1
- package/dist/callPhases.js +82 -0
- package/dist/cli.js +10 -1
- package/dist/context/rowStore.js +1 -1
- package/dist/conversation/parser.js +40 -36
- package/dist/db/client.js +6 -4
- package/dist/db/handoffTable.js +31 -0
- package/dist/db/indexSalvage.js +24 -4
- package/dist/db/migrations.js +30 -2
- package/dist/db/sqlite.js +396 -0
- package/dist/digests/digestCache.js +6 -2
- package/dist/digests/digests.js +3 -3
- package/dist/digests/worker.js +3 -1
- package/dist/embeddings/caching.js +5 -5
- package/dist/embeddings/local.js +5 -3
- package/dist/embeddings/openai.js +4 -3
- package/dist/embeddings/simdScan.js +3 -1
- package/dist/embeddings/vectorIndex.js +0 -1
- package/dist/embeddings/vertex.js +123 -13
- package/dist/eval/cacheReport.js +7 -0
- package/dist/eval/context700-candidate.js +5 -5
- package/dist/eval/evalCache.js +46 -3
- package/dist/eval/indexSnapshots.js +548 -0
- package/dist/eval/pageDump.js +67 -0
- package/dist/eval/runManifest.js +6 -0
- package/dist/eval/textTokens.js +17 -0
- package/dist/evidence/attach.js +0 -3
- package/dist/evidence/evidenceCache.js +3 -3
- package/dist/evidence/options.js +46 -25
- package/dist/evidence/secondPass.js +8 -170
- package/dist/extraction/factExtractor.js +1 -1
- package/dist/extraction/preferenceExtractor.js +1 -4
- package/dist/extraction/typedFacts.js +1 -1
- package/dist/importing/apply.js +42 -14
- package/dist/importing/cli.js +9 -8
- package/dist/importing/existing.js +2 -0
- package/dist/importing/facts.js +0 -2
- package/dist/importing/plan.js +0 -5
- package/dist/importing/source.js +1 -1
- package/dist/indexer/parser.js +4 -1
- package/dist/indexer/pipeline.js +99 -279
- package/dist/indexer/walker.js +11 -25
- package/dist/mcp/server.js +36 -6
- package/dist/memory/handoff.js +363 -0
- package/dist/memory/lifecycle.js +310 -75
- package/dist/memory/sourceName.js +77 -1
- package/dist/memory/statusFacts.js +61 -0
- package/dist/memorymap/memoryMap.js +39 -13
- package/dist/memorymap/replacements.js +8 -3
- package/dist/mvp/agentMemory.js +106 -50
- package/dist/mvp/answerCheck.js +211 -0
- package/dist/mvp/cli.js +3 -1
- package/dist/mvp/compileCache.js +3 -10
- package/dist/mvp/compileSeed.js +9 -3
- package/dist/mvp/contextDistinct.js +53 -0
- package/dist/mvp/contextProfile.js +87 -0
- package/dist/mvp/directorySurface.js +68 -0
- package/dist/mvp/engine.js +39 -26
- package/dist/mvp/exportAll.js +17 -2
- package/dist/mvp/factMarkers.js +18 -2
- package/dist/mvp/history.js +5 -4
- package/dist/mvp/log.js +118 -0
- package/dist/mvp/server.js +642 -127
- package/dist/mvp/serverCard.js +12 -3
- package/dist/mvp/telemetry.js +35 -2
- package/dist/net/poolFetch.js +174 -0
- package/dist/recall/actionRecall.js +282 -0
- package/dist/recall/autoRecall.js +447 -0
- package/dist/recall/searchRecall.js +189 -0
- package/dist/retrieval/entityLookup.js +53 -4
- package/dist/retrieval/orchestrator.js +93 -8
- package/dist/retrieval/plainWording.js +236 -0
- package/dist/retrieval/readerLayout.js +143 -224
- package/dist/router/routerCache.js +2 -2
- package/dist/watcher/watcher.js +5 -3
- package/package.json +5 -7
- package/dist/importing/index.js +0 -29
|
@@ -1,39 +1,50 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.READER_SESSION_CAP = exports.LAYOUT_MIN_TURNS = void 0;
|
|
4
4
|
exports.readerLayoutEnabled = readerLayoutEnabled;
|
|
5
5
|
exports.readerLayoutSkipsAdvice = readerLayoutSkipsAdvice;
|
|
6
6
|
exports.readerLayoutEnabledFor = readerLayoutEnabledFor;
|
|
7
7
|
exports.readerTimelineEnabled = readerTimelineEnabled;
|
|
8
|
-
exports.roleAwarePageEnabled = roleAwarePageEnabled;
|
|
9
|
-
exports.refersToAssistant = refersToAssistant;
|
|
10
|
-
exports.clipWords = clipWords;
|
|
11
|
-
exports.clipAssistantTurns = clipAssistantTurns;
|
|
12
|
-
exports.clipAssistantHit = clipAssistantHit;
|
|
13
|
-
exports.clipAssistantChunks = clipAssistantChunks;
|
|
14
8
|
exports.isCurrentValueQuestion = isCurrentValueQuestion;
|
|
15
9
|
exports.currentValueTimeline = currentValueTimeline;
|
|
16
10
|
exports.isConversation = isConversation;
|
|
17
11
|
exports.conversationNoteMatcher = conversationNoteMatcher;
|
|
12
|
+
exports.readerSessionCap = readerSessionCap;
|
|
13
|
+
exports.readerBlockOrder = readerBlockOrder;
|
|
14
|
+
exports.isHeaderOnly = isHeaderOnly;
|
|
18
15
|
exports.layoutHitsForReader = layoutHitsForReader;
|
|
16
|
+
exports.layoutPage = layoutPage;
|
|
17
|
+
exports.readerLayoutPage = readerLayoutPage;
|
|
19
18
|
exports.readerLayoutOf = readerLayoutOf;
|
|
20
19
|
const relativeTimeHints_1 = require("./relativeTimeHints");
|
|
20
|
+
const orchestrator_1 = require("./orchestrator");
|
|
21
21
|
const questionSpan_1 = require("./questionSpan");
|
|
22
22
|
const readerEvents_1 = require("./readerEvents");
|
|
23
23
|
const conversationShape_1 = require("./conversationShape");
|
|
24
24
|
const adviceQuestion_1 = require("./adviceQuestion");
|
|
25
25
|
/**
|
|
26
26
|
* How a search reply is laid out for the model that reads it. Retrieval picks the window and orders it; this only
|
|
27
|
-
* changes how what it picked is handed over, and it never
|
|
27
|
+
* changes how what it picked is handed over, and it never cuts or rewrites a word of an excerpt it shows:
|
|
28
28
|
*
|
|
29
29
|
* - one hit per session: the chunks of a note on one date, which the newest-first layout left apart (a session's header,
|
|
30
30
|
* which carries its date, could come after the turns it dates, and turns of other sessions in between), come as one hit,
|
|
31
31
|
* in the note's own order, so every turn sits under the date of its session;
|
|
32
|
+
* - a session whose only chunk is its header (`# Session x` and a `Timestamp:` line) is no hit: it dated nothing the reader
|
|
33
|
+
* can see and took a place in the reply (a third of the hits of the 2026-10-16 LongMemEval-S run);
|
|
34
|
+
* - the blocks stand where the page had them (newest first by default). RENOR_READER_ORDER=relevance puts them best match
|
|
35
|
+
* first instead (a block's score is its best chunk's, ties newest first) and the reply says so (`rankedBy: score`,
|
|
36
|
+
* readerLayoutPage; only a page whose ordinary notes are all conversations). It is off by default: on RenorBench v2 it puts a
|
|
37
|
+
* superseded value ahead of the current one (tests/eval/renorbench-v2.test.ts);
|
|
38
|
+
* - RENOR_READER_SESSION_CAP=N lets a session show at most N chunks besides its header, except that every chunk the query
|
|
39
|
+
* matched stays: the cap only stops the statements the session expansion added. Off by default (0): on RenorBench v2 a cap
|
|
40
|
+
* of 10 drops the remark a question asks about, which only the expansion delivers;
|
|
32
41
|
* - a session that reaches the reader without its header (the date line is its first chunk) opens with
|
|
33
42
|
* `[Session date: <date>]`;
|
|
34
43
|
* - each time phrase that names one point in time ("three weeks ago", "last Thursday") is resolved to its date below
|
|
35
44
|
* the excerpt (relativeTimeHints.ts).
|
|
36
45
|
*
|
|
46
|
+
* Whole chunks leave (a header stub, a statement over the cap); no chunk is cut or reworded.
|
|
47
|
+
*
|
|
37
48
|
* Only notes shaped like a conversation (conversationShape.ts) are laid out: a session is one conversation, and merging
|
|
38
49
|
* the chunks of a document or the entries of a daily note would break "a hit is one memory".
|
|
39
50
|
*
|
|
@@ -51,10 +62,6 @@ const adviceQuestion_1 = require("./adviceQuestion");
|
|
|
51
62
|
* Oldest first, the latest line marked, earlier lines kept. It quotes those turns and does not cut or reword an excerpt,
|
|
52
63
|
* invent a value, or do arithmetic on a session header (a header date orders two different calendar days only; a shared
|
|
53
64
|
* day is not a clock). A duration question or an order question is left untouched.
|
|
54
|
-
*
|
|
55
|
-
* Experimental, opt-in, and only on top of this layout (`RENOR_ROLE_AWARE_PAGE=1`; default off, so a layout-on page is
|
|
56
|
-
* byte-identical): a role-aware page. Unless the question refers to what the assistant said, each assistant turn on the
|
|
57
|
-
* page is clipped to its first 60 words; user turns, headings and dates are never touched (see roleAwarePage below).
|
|
58
65
|
*/
|
|
59
66
|
function readerLayoutEnabled() {
|
|
60
67
|
const v = (process.env.RENOR_READER_LAYOUT ?? '').trim().toLowerCase();
|
|
@@ -78,149 +85,6 @@ function readerTimelineEnabled(env = process.env) {
|
|
|
78
85
|
const v = (env.RENOR_READER_TIMELINE ?? '').trim().toLowerCase();
|
|
79
86
|
return v === '1' || v === 'true' || v === 'yes' || v === 'on';
|
|
80
87
|
}
|
|
81
|
-
/**
|
|
82
|
-
* RENOR_ROLE_AWARE_PAGE (default off, experimental; `1` / `true` / `yes` / `on` turns it on): a role-aware page. When the
|
|
83
|
-
* question does not refer to what the assistant said, each assistant turn of a conversation on the page is clipped to its
|
|
84
|
-
* first `ROLE_AWARE_ASSISTANT_WORDS` words, cut at a word boundary and marked "…". User turns, every heading, a session's
|
|
85
|
-
* header and `Timestamp:` line, the `[Session date: …]` and `[Counted from this session's date …]` lines and the hit's
|
|
86
|
-
* `session_date` are never touched. Off, the page is not looked at, so it is byte for byte the former one.
|
|
87
|
-
*
|
|
88
|
-
* Why (evidence analysis of the 2026-10-18 official LongMemEval run, not yet an answer measurement): in the multi-session,
|
|
89
|
-
* temporal, knowledge-update and preference questions the evidence is always a user turn, while assistant text is 28-39%
|
|
90
|
-
* of the page's words; clipping assistant turns to 60 words on those questions removes 21% of them. A question whose
|
|
91
|
-
* evidence is what the assistant said is kept whole by `refersToAssistant`.
|
|
92
|
-
*
|
|
93
|
-
* The speaker comes only from the turn heading (`### Turn N — Assistant`), never from a benchmark label. A hit that starts
|
|
94
|
-
* inside an assistant turn (the later chunk of a long answer, with no heading of its own) is known by its heading path.
|
|
95
|
-
*/
|
|
96
|
-
exports.ROLE_AWARE_ASSISTANT_WORDS = 60;
|
|
97
|
-
function roleAwarePageEnabled(env = process.env) {
|
|
98
|
-
const v = (env.RENOR_ROLE_AWARE_PAGE ?? '').trim().toLowerCase();
|
|
99
|
-
return v === '1' || v === 'true' || v === 'yes' || v === 'on';
|
|
100
|
-
}
|
|
101
|
-
/** Phrasing that asks about what the assistant said; such a question keeps assistant turns whole. */
|
|
102
|
-
const ASSISTANT_REFERENCE = new RegExp([
|
|
103
|
-
String.raw `\byou (?:had |just |once )?(?:recommended|suggested|told me|said|mentioned|gave me|provided|listed)\b`,
|
|
104
|
-
String.raw `\bdid you (?:recommend|suggest|tell me|say|mention|give me|provide|list)\b`,
|
|
105
|
-
String.raw `\bour (?:previous|last|earlier|past|prior) (?:conversation|chat|discussion)s?\b`,
|
|
106
|
-
String.raw `\bremind me (?:again )?(?:what|which|of what|how|where|who) you\b`,
|
|
107
|
-
String.raw `\bearlier,? you\b`,
|
|
108
|
-
].join('|'), 'i');
|
|
109
|
-
function refersToAssistant(query) {
|
|
110
|
-
return ASSISTANT_REFERENCE.test(query.replace(/[\u2018\u2019]/g, "'"));
|
|
111
|
-
}
|
|
112
|
-
const LAYOUT_HEADING = /^(#{1,6})[ \t]+(.*?)[ \t]*\r?$/;
|
|
113
|
-
const TURN_ROLE = /^Turn[ \t]+\d+[ \t]+[—–-][ \t]+(\S.*)$/i;
|
|
114
|
-
/** Lines the layout itself writes, which date what is around them: kept whole, and their words are not the turn's. */
|
|
115
|
-
const DATE_LINE = /^\[(?:Session date: |Counted from this session's date)/;
|
|
116
|
-
/** For a heading's text: 'assistant' or 'other' when it is a turn heading, null when it is not one. */
|
|
117
|
-
function turnRole(heading) {
|
|
118
|
-
const role = TURN_ROLE.exec(heading.trim())?.[1];
|
|
119
|
-
if (role === undefined)
|
|
120
|
-
return null;
|
|
121
|
-
return /^assistant$/i.test(role.trim()) ? 'assistant' : 'other';
|
|
122
|
-
}
|
|
123
|
-
/** At most `budget` words of `body`, cut at a word boundary and marked "…"; its leading and trailing whitespace is kept. */
|
|
124
|
-
function takeWords(body, budget) {
|
|
125
|
-
const words = [...body.matchAll(/\S+/g)];
|
|
126
|
-
if (words.length <= budget)
|
|
127
|
-
return { text: body, used: words.length };
|
|
128
|
-
const end = budget === 0 ? (/^\s*/.exec(body)?.[0].length ?? 0) : (words[budget - 1].index ?? 0) + words[budget - 1][0].length;
|
|
129
|
-
const trailing = /\s*$/.exec(body)?.[0] ?? '';
|
|
130
|
-
return { text: `${body.slice(0, end)}${budget === 0 ? '' : ' '}…${trailing}`, used: budget };
|
|
131
|
-
}
|
|
132
|
-
/** The first `limit` words of `body`, cut at a word boundary and marked " …"; a body that fits is returned as is. */
|
|
133
|
-
function clipWords(body, limit = exports.ROLE_AWARE_ASSISTANT_WORDS) {
|
|
134
|
-
return takeWords(body, limit).text;
|
|
135
|
-
}
|
|
136
|
-
/** What is left of an assistant turn's words at the end of a text: null when the text ends outside an assistant turn. */
|
|
137
|
-
function clipFrom(text, limit, start) {
|
|
138
|
-
const out = [];
|
|
139
|
-
let budget = start;
|
|
140
|
-
let body = [];
|
|
141
|
-
const flush = () => {
|
|
142
|
-
if (body.length === 0)
|
|
143
|
-
return;
|
|
144
|
-
const joined = body.join('\n');
|
|
145
|
-
if (budget === null)
|
|
146
|
-
out.push(joined);
|
|
147
|
-
else {
|
|
148
|
-
const taken = takeWords(joined, budget);
|
|
149
|
-
budget -= taken.used;
|
|
150
|
-
out.push(taken.text);
|
|
151
|
-
}
|
|
152
|
-
body = [];
|
|
153
|
-
};
|
|
154
|
-
for (const line of text.split('\n')) {
|
|
155
|
-
if (DATE_LINE.test(line)) {
|
|
156
|
-
flush();
|
|
157
|
-
out.push(line);
|
|
158
|
-
continue;
|
|
159
|
-
}
|
|
160
|
-
const heading = LAYOUT_HEADING.exec(line);
|
|
161
|
-
if (!heading) {
|
|
162
|
-
body.push(line);
|
|
163
|
-
continue;
|
|
164
|
-
}
|
|
165
|
-
flush();
|
|
166
|
-
const role = turnRole(heading[2]);
|
|
167
|
-
if (role !== null)
|
|
168
|
-
budget = role === 'assistant' ? limit : null;
|
|
169
|
-
else if (heading[1].length <= 2)
|
|
170
|
-
budget = null;
|
|
171
|
-
out.push(line);
|
|
172
|
-
}
|
|
173
|
-
flush();
|
|
174
|
-
// Each piece was one or more whole lines; joining them with '\n' restores the line breaks between them.
|
|
175
|
-
return { text: out.join('\n'), budget };
|
|
176
|
-
}
|
|
177
|
-
/**
|
|
178
|
-
* Clip each assistant turn of `text` to its first `limit` words. A turn runs from its `### Turn N — Assistant` heading to
|
|
179
|
-
* the next turn heading (or the next `#`/`##` heading); a smaller heading inside it is kept and its words count toward
|
|
180
|
-
* the same turn. Every heading line, every date line and everything outside an assistant turn (a user turn, a session
|
|
181
|
-
* header) is left byte for byte. `inAssistant`: the text starts inside an assistant turn.
|
|
182
|
-
*/
|
|
183
|
-
function clipAssistantTurns(text, limit = exports.ROLE_AWARE_ASSISTANT_WORDS, inAssistant = false) {
|
|
184
|
-
return clipFrom(text, limit, inAssistant ? limit : null).text;
|
|
185
|
-
}
|
|
186
|
-
function headingPathOf(hit) {
|
|
187
|
-
const value = hit.headingPath;
|
|
188
|
-
return typeof value === 'string' ? value : '';
|
|
189
|
-
}
|
|
190
|
-
function startsInAssistantTurn(headingPath) {
|
|
191
|
-
return headingPath.split(' > ').map(turnRole).filter(role => role !== null).pop() === 'assistant';
|
|
192
|
-
}
|
|
193
|
-
/** One hit on its own: its assistant turns clipped (its heading path says whether it starts inside one), every other field as it was. */
|
|
194
|
-
function clipAssistantHit(hit, limit = exports.ROLE_AWARE_ASSISTANT_WORDS) {
|
|
195
|
-
const text = clipAssistantTurns(hit.text, limit, startsInAssistantTurn(headingPathOf(hit)));
|
|
196
|
-
return text === hit.text ? hit : { ...hit, text };
|
|
197
|
-
}
|
|
198
|
-
/**
|
|
199
|
-
* The chunks of a page with their assistant turns clipped, before the layout joins them. A long answer is stored as several
|
|
200
|
-
* chunks and its heading as one more, which may not reach the page; so each chunk is read with its own heading path, in
|
|
201
|
-
* its note's order, and a turn that runs on into the next chunk carries the words it has left: the turn, not each chunk,
|
|
202
|
-
* keeps its first `limit` words. Hits that are no chunk, and their order, are untouched.
|
|
203
|
-
*/
|
|
204
|
-
function clipAssistantChunks(hits, chunk, limit = exports.ROLE_AWARE_ASSISTANT_WORDS) {
|
|
205
|
-
const order = hits.map((_, i) => i).filter(i => chunk(hits[i]))
|
|
206
|
-
.sort((a, b) => (hits[a].notePath < hits[b].notePath ? -1 : hits[a].notePath > hits[b].notePath ? 1 : (hits[a].chunkId ?? 0) - (hits[b].chunkId ?? 0)));
|
|
207
|
-
const left = new Map();
|
|
208
|
-
const out = hits.slice();
|
|
209
|
-
for (const i of order) {
|
|
210
|
-
const hit = hits[i];
|
|
211
|
-
const headingPath = headingPathOf(hit);
|
|
212
|
-
const key = `${hit.notePath}\u0000${headingPath}`;
|
|
213
|
-
const start = startsInAssistantTurn(headingPath) ? (left.get(key) ?? limit) : null;
|
|
214
|
-
const clipped = clipFrom(hit.text, limit, start);
|
|
215
|
-
if (clipped.budget === null)
|
|
216
|
-
left.delete(key);
|
|
217
|
-
else
|
|
218
|
-
left.set(key, clipped.budget);
|
|
219
|
-
if (clipped.text !== hit.text)
|
|
220
|
-
out[i] = { ...hit, text: clipped.text };
|
|
221
|
-
}
|
|
222
|
-
return out;
|
|
223
|
-
}
|
|
224
88
|
// A duration or an order stays byte-identical. "Ago" and "last Tuesday" are a point in the past, not a changed value.
|
|
225
89
|
const DURATION_QUESTION = /\b(?:how (?:many|much) (?:days|weeks|months|years)|how long|how much time|how old)\b|\b(?:days|weeks|months|years)\b[^?]{0,24}\b(?:ago|passed)\b|\bago\b|\bpassed between\b|\blast (?:week|month|year|weekend|monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b/i;
|
|
226
90
|
const ORDER_QUESTION = /\border\b|\b(?:first to last|earliest to latest|happened first|came first)\b|\b(?:which|who)\b[^?]{0,80}\bfirst\b/i;
|
|
@@ -408,96 +272,151 @@ function conversationNoteMatcher(db) {
|
|
|
408
272
|
function isNoteChunk(hit, isLedger) {
|
|
409
273
|
return (hit.chunkId ?? 0) > 0 && hit.notePath !== '' && !isLedger(hit.notePath) && hit.author === undefined && hit.replaced === undefined;
|
|
410
274
|
}
|
|
275
|
+
/** The most chunks of one session's block (its header aside); RENOR_READER_SESSION_CAP, 0 (the default) is no cap. */
|
|
276
|
+
exports.READER_SESSION_CAP = 0;
|
|
277
|
+
function readerSessionCap(env = process.env) {
|
|
278
|
+
const raw = (env.RENOR_READER_SESSION_CAP ?? '').trim();
|
|
279
|
+
if (raw === '')
|
|
280
|
+
return exports.READER_SESSION_CAP;
|
|
281
|
+
const n = Number(raw);
|
|
282
|
+
return Number.isInteger(n) && n >= 0 ? n : exports.READER_SESSION_CAP;
|
|
283
|
+
}
|
|
284
|
+
/** Blocks keep the page's own (newest-first) order; RENOR_READER_ORDER=relevance puts them best match first. */
|
|
285
|
+
function readerBlockOrder(env = process.env) {
|
|
286
|
+
return (env.RENOR_READER_ORDER ?? '').trim().toLowerCase() === 'relevance' ? 'relevance' : 'date';
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* A chunk that is only a session's header: headings and `Timestamp:` lines, no turn. It dates the turns under it and says
|
|
290
|
+
* nothing itself; on its own it is a stub that takes a place in the reply (a third of the hits of the 2026-10-16 run).
|
|
291
|
+
*/
|
|
292
|
+
function isHeaderOnly(text) {
|
|
293
|
+
const lines = text.split(/\r?\n/).map(line => line.trim()).filter(line => line !== '' && !/^#{1,6}\s/.test(line));
|
|
294
|
+
return lines.length > 0 && lines.every(line => /^Timestamp:/i.test(line));
|
|
295
|
+
}
|
|
296
|
+
/** A hit of an ordinary note (a chunk the layout could place, a daily entry, a document paragraph): not a fact, a ledger event, a correction or the evidence and digest tables (`_renor/…`). */
|
|
297
|
+
function isOrdinaryNoteHit(hit, isLedger) {
|
|
298
|
+
return hit.notePath !== '' && !hit.notePath.startsWith('_renor/') && !isLedger(hit.notePath) && hit.author === undefined && hit.replaced === undefined;
|
|
299
|
+
}
|
|
411
300
|
function layoutHitsForReader(hits, opts) {
|
|
301
|
+
return layoutPage(hits, opts).hits;
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* `byRelevance` is true when the blocks were placed best match first, which the reply then says (`rankedBy: score`). It is
|
|
305
|
+
* done only to a page whose every chunk of an ordinary note is a conversation's: a daily note's entries or a document's
|
|
306
|
+
* paragraphs on the same page keep the order the page gave them, and a page that is half one order and half the other must
|
|
307
|
+
* not claim either key.
|
|
308
|
+
*/
|
|
309
|
+
function layoutPage(hits, opts) {
|
|
412
310
|
const chunk = (hit) => isNoteChunk(hit, opts.isLedger) && (opts.isConversation?.(hit.notePath) ?? true);
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
const
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
311
|
+
const cap = opts.sessionCap ?? readerSessionCap();
|
|
312
|
+
const order = opts.order ?? readerBlockOrder();
|
|
313
|
+
const laid = hits.map(hit => (chunk(hit) ? { ...hit, text: (0, relativeTimeHints_1.withRelativeTimeHints)(hit.text, hit.session_date) } : hit));
|
|
314
|
+
// One block per conversation on one date: every chunk of it under that date, wherever the page had put it.
|
|
315
|
+
const keyOf = (hit) => (chunk(hit) ? `${hit.notePath}\0${hit.session_date ?? ''}` : null);
|
|
316
|
+
const blocks = new Map();
|
|
317
|
+
laid.forEach((hit, index) => {
|
|
318
|
+
const key = keyOf(hit);
|
|
319
|
+
if (key === null)
|
|
320
|
+
return;
|
|
321
|
+
const block = blocks.get(key) ?? { header: [], body: [], first: index };
|
|
322
|
+
(isHeaderOnly(hit.text) ? block.header : block.body).push(hit);
|
|
323
|
+
blocks.set(key, block);
|
|
324
|
+
});
|
|
325
|
+
const made = new Map();
|
|
326
|
+
for (const [key, block] of blocks)
|
|
327
|
+
made.set(key, blockHit(block, cap));
|
|
328
|
+
// The blocks stand in the places the page gave its chunks; anything else keeps its place.
|
|
329
|
+
const slots = new Set(laid.flatMap((hit, index) => (keyOf(hit) !== null && blocks.get(keyOf(hit)).first === index ? [index] : [])));
|
|
330
|
+
let placed = [...slots].map(index => keyOf(laid[index])).filter(key => made.get(key) !== null);
|
|
331
|
+
const byRelevance = order === 'relevance' && placed.length > 0 && hits.every(hit => !isOrdinaryNoteHit(hit, opts.isLedger) || chunk(hit));
|
|
332
|
+
if (byRelevance) {
|
|
333
|
+
const at = new Map(placed.map((key, i) => [key, i]));
|
|
334
|
+
placed = [...placed].sort((a, b) => made.get(b).score - made.get(a).score
|
|
335
|
+
|| ((0, orchestrator_1.sessionDateMs)(made.get(b).session_date) || 0) - ((0, orchestrator_1.sessionDateMs)(made.get(a).session_date) || 0)
|
|
336
|
+
|| at.get(a) - at.get(b));
|
|
337
|
+
}
|
|
338
|
+
const out = [];
|
|
339
|
+
let next = 0;
|
|
340
|
+
laid.forEach((hit, index) => {
|
|
341
|
+
const key = keyOf(hit);
|
|
342
|
+
if (key === null)
|
|
343
|
+
out.push(hit);
|
|
344
|
+
else if (slots.has(index) && made.get(key) !== null)
|
|
345
|
+
out.push(made.get(placed[next++]));
|
|
346
|
+
});
|
|
347
|
+
const dated = out.map(hit => (chunk(hit) && hit.session_date && !/Timestamp:/i.test(hit.text)
|
|
421
348
|
? { ...hit, text: `[Session date: ${hit.session_date}]\n${hit.text}` }
|
|
422
349
|
: hit));
|
|
423
350
|
// RENOR_READER_DURATION (default on; `0` switches it off): one line of dates the user wrote, first, for a "how many days between" question (questionSpan.ts).
|
|
424
351
|
// It reads the chunks as they were found: joined, a turn's continuation would sit under the speaker before it.
|
|
425
352
|
// RENOR_READER_EVENTS (default on; `0` switches it off) adds event lines for an order or a date question; its gate leaves a duration question alone, so at most one of the two lines is added.
|
|
426
353
|
// RENOR_READER_TIMELINE (default off): a current-value timeline for a "what is it now" question; a duration or an order question is left alone.
|
|
427
|
-
|
|
354
|
+
// Each such line comes first, outside the order rankedBy names, as the evidence hit does. The event lines too read the
|
|
355
|
+
// conversations as the page found them: a session that came as its header alone is no hit, but its turns are still read.
|
|
356
|
+
return {
|
|
357
|
+
hits: withCurrentValueTimeline(withEventLines((0, questionSpan_1.withQuestionSpan)(dated, opts.query, { source: hits }), hits.filter(chunk), opts), opts.query),
|
|
358
|
+
byRelevance,
|
|
359
|
+
};
|
|
428
360
|
}
|
|
429
|
-
/** The event lines, as the first hit (reason "events"), when the flag is on and the question asks for an order or a date. */
|
|
430
|
-
function withEventLines(hits,
|
|
361
|
+
/** The event lines, as the first hit (reason "events"), when the flag is on and the question asks for an order or a date. `sources` are the conversation chunks they read. */
|
|
362
|
+
function withEventLines(hits, sources, opts) {
|
|
431
363
|
if (!(0, readerEvents_1.readerEventsEnabled)() || opts.query === undefined || opts.turnsOf === undefined || opts.query.trim() === '')
|
|
432
364
|
return hits;
|
|
433
|
-
const text = (0, readerEvents_1.eventLines)(
|
|
365
|
+
const text = (0, readerEvents_1.eventLines)(sources, opts.query, opts.turnsOf, { scoreTurns: opts.scoreTurns });
|
|
434
366
|
if (text === null)
|
|
435
367
|
return hits;
|
|
436
368
|
return [{ notePath: '', text, score: 0, reason: 'events' }, ...hits];
|
|
437
369
|
}
|
|
438
|
-
/**
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
const
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
}
|
|
455
|
-
if (placed.has(key))
|
|
456
|
-
continue;
|
|
457
|
-
placed.add(key);
|
|
458
|
-
out.push(...groups.get(key).sort((a, b) => (a.chunkId ?? 0) - (b.chunkId ?? 0)));
|
|
459
|
-
}
|
|
460
|
-
return out;
|
|
461
|
-
}
|
|
462
|
-
/** Neighbouring hits of one note and date become one: the texts in order, the score and reason of the best match. */
|
|
463
|
-
function mergeRuns(hits, chunk) {
|
|
464
|
-
const out = [];
|
|
465
|
-
let i = 0;
|
|
466
|
-
while (i < hits.length) {
|
|
467
|
-
const first = hits[i];
|
|
468
|
-
let j = i + 1;
|
|
469
|
-
if (chunk(first))
|
|
470
|
-
while (j < hits.length && chunk(hits[j]) && hits[j].notePath === first.notePath && hits[j].session_date === first.session_date)
|
|
471
|
-
j++;
|
|
472
|
-
const run = hits.slice(i, j);
|
|
473
|
-
if (run.length === 1)
|
|
474
|
-
out.push(first);
|
|
475
|
-
else {
|
|
476
|
-
const best = run.reduce((a, b) => (b.score > a.score ? b : a));
|
|
477
|
-
const similarities = run.flatMap(h => (typeof h.similarity === 'number' ? [h.similarity] : []));
|
|
478
|
-
out.push({
|
|
479
|
-
...first,
|
|
480
|
-
text: run.map(h => h.text).join('\n\n'),
|
|
481
|
-
score: best.score,
|
|
482
|
-
reason: best.reason,
|
|
483
|
-
...(similarities.length > 0 ? { similarity: Math.max(...similarities) } : {}),
|
|
484
|
-
});
|
|
485
|
-
}
|
|
486
|
-
i = j;
|
|
370
|
+
/**
|
|
371
|
+
* One conversation's hit: its header first when it came, then its turns in the conversation's own order, with the best
|
|
372
|
+
* score and reason of its chunks. A session that came as a header and nothing else is no hit. Over the cap, every chunk the
|
|
373
|
+
* query matched stays and the statements the session expansion added are taken in the order they came, as many as the cap leaves room for.
|
|
374
|
+
*/
|
|
375
|
+
function blockHit(block, cap) {
|
|
376
|
+
if (block.body.length === 0)
|
|
377
|
+
return null;
|
|
378
|
+
const byPlace = (a, b) => (a.chunkId ?? 0) - (b.chunkId ?? 0);
|
|
379
|
+
let body = block.body;
|
|
380
|
+
if (cap > 0 && body.length > cap) {
|
|
381
|
+
// What the query matched always stays; the cap only limits the statements the session expansion added.
|
|
382
|
+
const expanded = body.filter(h => h.reason === orchestrator_1.SESSION_EXPAND_REASON);
|
|
383
|
+
const room = Math.max(0, cap - (body.length - expanded.length));
|
|
384
|
+
const kept = new Set([...body.filter(h => h.reason !== orchestrator_1.SESSION_EXPAND_REASON), ...expanded.slice(0, room)]);
|
|
385
|
+
body = body.filter(h => kept.has(h));
|
|
487
386
|
}
|
|
488
|
-
|
|
387
|
+
const run = [...block.header, ...body].sort(byPlace);
|
|
388
|
+
if (run.length === 1)
|
|
389
|
+
return run[0];
|
|
390
|
+
const best = run.reduce((a, b) => (b.score > a.score ? b : a));
|
|
391
|
+
const similarities = run.flatMap(h => (typeof h.similarity === 'number' ? [h.similarity] : []));
|
|
392
|
+
return {
|
|
393
|
+
...run[0],
|
|
394
|
+
text: run.map(h => h.text).join('\n\n'),
|
|
395
|
+
score: best.score,
|
|
396
|
+
reason: best.reason,
|
|
397
|
+
...(similarities.length > 0 ? { similarity: Math.max(...similarities) } : {}),
|
|
398
|
+
};
|
|
489
399
|
}
|
|
490
400
|
/**
|
|
491
401
|
* The page search_memory shows, laid out for its reader when RENOR_READER_LAYOUT is on (the chunk ids the layout needed
|
|
492
|
-
* leave with it); otherwise the page as it was. `
|
|
402
|
+
* leave with it); otherwise the page as it was. `rankedBy` is the key the returned page is in order of. `enabled` is the
|
|
403
|
+
* answer for this query (readerLayoutEnabledFor).
|
|
493
404
|
*/
|
|
494
|
-
function
|
|
405
|
+
function readerLayoutPage(rankedBy, hits, isLedger, events = {}, enabled = readerLayoutEnabled()) {
|
|
495
406
|
if (!enabled)
|
|
496
|
-
return hits;
|
|
407
|
+
return { hits, rankedBy };
|
|
497
408
|
// A bare string is the question alone (the duration line's form); the object also carries what the event lines read.
|
|
498
409
|
const input = typeof events === 'string' ? { query: events } : events;
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
410
|
+
const laid = layoutPage(hits, { rankedBy, isLedger, ...input });
|
|
411
|
+
return {
|
|
412
|
+
hits: laid.hits.map(hit => {
|
|
413
|
+
const { chunkId: _chunkId, ...rest } = hit;
|
|
414
|
+
return rest;
|
|
415
|
+
}),
|
|
416
|
+
rankedBy: laid.byRelevance ? 'score' : rankedBy,
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
/** The hits of readerLayoutPage alone. */
|
|
420
|
+
function readerLayoutOf(rankedBy, hits, isLedger, events = {}, enabled = readerLayoutEnabled()) {
|
|
421
|
+
return readerLayoutPage(rankedBy, hits, isLedger, events, enabled).hits;
|
|
503
422
|
}
|
|
@@ -7,7 +7,7 @@ exports.RouterCache = exports.ROUTER_CACHE_SCHEMA_VERSION = void 0;
|
|
|
7
7
|
const node_fs_1 = __importDefault(require("node:fs"));
|
|
8
8
|
const node_crypto_1 = __importDefault(require("node:crypto"));
|
|
9
9
|
const node_path_1 = __importDefault(require("node:path"));
|
|
10
|
-
const
|
|
10
|
+
const sqlite_1 = __importDefault(require("../db/sqlite"));
|
|
11
11
|
/**
|
|
12
12
|
* Content-addressed cache for the session router (RENOR_SESSION_ROUTER, src/router/sessionRouter.ts): the
|
|
13
13
|
* per-note summaries the router reads, and the router's own answers.
|
|
@@ -34,7 +34,7 @@ class RouterCache {
|
|
|
34
34
|
touchRouteStmt;
|
|
35
35
|
constructor(dbPath) {
|
|
36
36
|
node_fs_1.default.mkdirSync(node_path_1.default.dirname(dbPath), { recursive: true });
|
|
37
|
-
this.db = new
|
|
37
|
+
this.db = new sqlite_1.default(dbPath);
|
|
38
38
|
this.db.pragma('journal_mode = WAL');
|
|
39
39
|
this.db.pragma('busy_timeout = 10000');
|
|
40
40
|
this.db.exec(`
|
package/dist/watcher/watcher.js
CHANGED
|
@@ -8,6 +8,8 @@ const chokidar_1 = __importDefault(require("chokidar"));
|
|
|
8
8
|
const fs_1 = __importDefault(require("fs"));
|
|
9
9
|
const path_1 = __importDefault(require("path"));
|
|
10
10
|
const pipeline_1 = require("../indexer/pipeline");
|
|
11
|
+
const log_1 = require("../mvp/log");
|
|
12
|
+
const log = (0, log_1.createLogger)('engine');
|
|
11
13
|
function loadIgnorePatterns(vaultPath) {
|
|
12
14
|
try {
|
|
13
15
|
const raw = fs_1.default.readFileSync(path_1.default.join(vaultPath, '.semanticignore'), 'utf8');
|
|
@@ -52,10 +54,10 @@ function startWatcher(vaultPath, db, options = {}) {
|
|
|
52
54
|
pending.delete(filePath);
|
|
53
55
|
try {
|
|
54
56
|
await (0, pipeline_1.indexFile)(db, filePath, options);
|
|
55
|
-
|
|
57
|
+
log.info('watcher_indexed', { file: path_1.default.relative(vaultPath, filePath) });
|
|
56
58
|
}
|
|
57
59
|
catch (err) {
|
|
58
|
-
|
|
60
|
+
log.error('watcher_index_failed', { file: path_1.default.relative(vaultPath, filePath) }, err);
|
|
59
61
|
}
|
|
60
62
|
}, 500));
|
|
61
63
|
};
|
|
@@ -69,7 +71,7 @@ function startWatcher(vaultPath, db, options = {}) {
|
|
|
69
71
|
db.prepare('DELETE FROM relations WHERE source_note = ?').run(filePath);
|
|
70
72
|
db.prepare('DELETE FROM notes WHERE path = ?').run(filePath);
|
|
71
73
|
})();
|
|
72
|
-
|
|
74
|
+
log.info('watcher_removed', { file: path_1.default.relative(vaultPath, filePath) });
|
|
73
75
|
});
|
|
74
76
|
return watcher;
|
|
75
77
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "renor-engine",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.1.0",
|
|
4
4
|
"description": "Local-first semantic memory — indexing, multi-signal retrieval, MCP",
|
|
5
5
|
"main": "dist/cli.js",
|
|
6
6
|
"files": [
|
|
@@ -64,18 +64,17 @@
|
|
|
64
64
|
"homepage": "https://renor.cloud",
|
|
65
65
|
"repository": {
|
|
66
66
|
"type": "git",
|
|
67
|
-
"url": "git+https://github.com/
|
|
67
|
+
"url": "git+https://github.com/renorai/renor-engine.git"
|
|
68
68
|
},
|
|
69
69
|
"bugs": {
|
|
70
|
-
"url": "https://github.com/
|
|
70
|
+
"url": "https://github.com/renorai/renor-engine/issues"
|
|
71
71
|
},
|
|
72
72
|
"author": "Patrick Passos",
|
|
73
|
-
"license": "
|
|
73
|
+
"license": "Apache-2.0",
|
|
74
74
|
"type": "commonjs",
|
|
75
75
|
"dependencies": {
|
|
76
76
|
"@modelcontextprotocol/sdk": "^1.31.0",
|
|
77
77
|
"@posthog/mcp": "0.21.0",
|
|
78
|
-
"better-sqlite3": "^12.8.0",
|
|
79
78
|
"chokidar": "^3.6.0",
|
|
80
79
|
"commander": "^14.0.3",
|
|
81
80
|
"express": "^5.2.1",
|
|
@@ -88,7 +87,6 @@
|
|
|
88
87
|
},
|
|
89
88
|
"devDependencies": {
|
|
90
89
|
"@xenova/transformers": "^2.17.2",
|
|
91
|
-
"@types/better-sqlite3": "^7.6.13",
|
|
92
90
|
"@types/express": "^5.0.6",
|
|
93
91
|
"@types/node": "^25.9.8",
|
|
94
92
|
"@types/supertest": "^7.2.0",
|
|
@@ -117,6 +115,6 @@
|
|
|
117
115
|
}
|
|
118
116
|
},
|
|
119
117
|
"engines": {
|
|
120
|
-
"node": "
|
|
118
|
+
"node": "^22.16.0 || >=24"
|
|
121
119
|
}
|
|
122
120
|
}
|
package/dist/importing/index.js
DELETED
|
@@ -1,29 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.validateCarriedFact = exports.tryNormalizeScope = exports.toCanonicalInstant = exports.deterministicEventId = exports.renderNoteMarkdown = exports.destinationRelPath = exports.readExistingStreams = exports.applyImportPlan = exports.streamKey = exports.contentHash = exports.buildImportPlan = exports.toRelative = exports.isSelected = exports.readSource = exports.readRenorIndexSource = exports.readBundleSource = exports.readMarkdownSource = exports.BUNDLE_FORMAT = void 0;
|
|
4
|
-
var types_1 = require("./types");
|
|
5
|
-
Object.defineProperty(exports, "BUNDLE_FORMAT", { enumerable: true, get: function () { return types_1.BUNDLE_FORMAT; } });
|
|
6
|
-
var markdown_1 = require("./markdown");
|
|
7
|
-
Object.defineProperty(exports, "readMarkdownSource", { enumerable: true, get: function () { return markdown_1.readMarkdownSource; } });
|
|
8
|
-
var source_1 = require("./source");
|
|
9
|
-
Object.defineProperty(exports, "readBundleSource", { enumerable: true, get: function () { return source_1.readBundleSource; } });
|
|
10
|
-
Object.defineProperty(exports, "readRenorIndexSource", { enumerable: true, get: function () { return source_1.readRenorIndexSource; } });
|
|
11
|
-
Object.defineProperty(exports, "readSource", { enumerable: true, get: function () { return source_1.readSource; } });
|
|
12
|
-
Object.defineProperty(exports, "isSelected", { enumerable: true, get: function () { return source_1.isSelected; } });
|
|
13
|
-
Object.defineProperty(exports, "toRelative", { enumerable: true, get: function () { return source_1.toRelative; } });
|
|
14
|
-
var plan_1 = require("./plan");
|
|
15
|
-
Object.defineProperty(exports, "buildImportPlan", { enumerable: true, get: function () { return plan_1.buildImportPlan; } });
|
|
16
|
-
Object.defineProperty(exports, "contentHash", { enumerable: true, get: function () { return plan_1.contentHash; } });
|
|
17
|
-
Object.defineProperty(exports, "streamKey", { enumerable: true, get: function () { return plan_1.streamKey; } });
|
|
18
|
-
var apply_1 = require("./apply");
|
|
19
|
-
Object.defineProperty(exports, "applyImportPlan", { enumerable: true, get: function () { return apply_1.applyImportPlan; } });
|
|
20
|
-
var existing_1 = require("./existing");
|
|
21
|
-
Object.defineProperty(exports, "readExistingStreams", { enumerable: true, get: function () { return existing_1.readExistingStreams; } });
|
|
22
|
-
var serialize_1 = require("./serialize");
|
|
23
|
-
Object.defineProperty(exports, "destinationRelPath", { enumerable: true, get: function () { return serialize_1.destinationRelPath; } });
|
|
24
|
-
Object.defineProperty(exports, "renderNoteMarkdown", { enumerable: true, get: function () { return serialize_1.renderNoteMarkdown; } });
|
|
25
|
-
var facts_1 = require("./facts");
|
|
26
|
-
Object.defineProperty(exports, "deterministicEventId", { enumerable: true, get: function () { return facts_1.deterministicEventId; } });
|
|
27
|
-
Object.defineProperty(exports, "toCanonicalInstant", { enumerable: true, get: function () { return facts_1.toCanonicalInstant; } });
|
|
28
|
-
Object.defineProperty(exports, "tryNormalizeScope", { enumerable: true, get: function () { return facts_1.tryNormalizeScope; } });
|
|
29
|
-
Object.defineProperty(exports, "validateCarriedFact", { enumerable: true, get: function () { return facts_1.validateCarriedFact; } });
|