renor-engine 0.0.0-stage → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +75 -0
- package/README.md +105 -2
- package/dist/.tsbuildinfo +1 -0
- package/dist/BUILD.json +6 -0
- package/dist/api/apiAuth.js +43 -0
- package/dist/api/server.js +174 -0
- package/dist/buildIdentity.js +149 -0
- package/dist/callPhases.js +82 -0
- package/dist/cli-extractors.js +46 -0
- package/dist/cli.js +500 -0
- package/dist/config.js +65 -0
- package/dist/context/composer.js +320 -0
- package/dist/context/readHonesty.js +478 -0
- package/dist/context/rowStore.js +238 -0
- package/dist/conversation/formatter.js +81 -0
- package/dist/conversation/parser.js +193 -0
- package/dist/conversation/types.js +2 -0
- package/dist/conversation/writer.js +42 -0
- package/dist/db/client.js +149 -0
- package/dist/db/handoffTable.js +31 -0
- package/dist/db/indexSalvage.js +101 -0
- package/dist/db/migrations.js +670 -0
- package/dist/db/readCache.js +109 -0
- package/dist/db/schema.js +5 -0
- package/dist/db/sqlite.js +396 -0
- package/dist/db/statements.js +89 -0
- package/dist/digests/digestCache.js +146 -0
- package/dist/digests/digests.js +597 -0
- package/dist/digests/worker.js +350 -0
- package/dist/embeddings/caching.js +337 -0
- package/dist/embeddings/coalescing.js +177 -0
- package/dist/embeddings/interactive.js +103 -0
- package/dist/embeddings/local.js +70 -0
- package/dist/embeddings/openai.js +106 -0
- package/dist/embeddings/rebuild.js +166 -0
- package/dist/embeddings/simdScan.js +260 -0
- package/dist/embeddings/space.js +21 -0
- package/dist/embeddings/types.js +14 -0
- package/dist/embeddings/vectorIndex.js +674 -0
- package/dist/embeddings/vertex.js +334 -0
- package/dist/eval/artifacts.js +42 -0
- package/dist/eval/cacheReport.js +146 -0
- package/dist/eval/comparison.js +182 -0
- package/dist/eval/context700-candidate.js +145 -0
- package/dist/eval/evalCache.js +568 -0
- package/dist/eval/hashEmbedding.js +35 -0
- package/dist/eval/indexSnapshots.js +548 -0
- package/dist/eval/longmemeval.js +56 -0
- package/dist/eval/metrics.js +181 -0
- package/dist/eval/pageDump.js +67 -0
- package/dist/eval/renorbenchBaseline.js +486 -0
- package/dist/eval/renorbenchDocuments.js +134 -0
- package/dist/eval/renorbenchProduct.js +182 -0
- package/dist/eval/runLedger.js +279 -0
- package/dist/eval/runManifest.js +485 -0
- package/dist/eval/scoring.js +197 -0
- package/dist/eval/textTokens.js +17 -0
- package/dist/evidence/attach.js +264 -0
- package/dist/evidence/background.js +147 -0
- package/dist/evidence/evidenceCache.js +182 -0
- package/dist/evidence/gate.js +37 -0
- package/dist/evidence/keyPool.js +85 -0
- package/dist/evidence/ollamaChat.js +84 -0
- package/dist/evidence/optin.js +165 -0
- package/dist/evidence/options.js +151 -0
- package/dist/evidence/secondPass.js +294 -0
- package/dist/extraction/classifyCapture.js +34 -0
- package/dist/extraction/decision-model.js +174 -0
- package/dist/extraction/eventExtractor.js +113 -0
- package/dist/extraction/extractionClient.js +122 -0
- package/dist/extraction/factExtractor.js +46 -0
- package/dist/extraction/factsWritePath.js +179 -0
- package/dist/extraction/preferenceExtractor.js +150 -0
- package/dist/extraction/secretGuard.js +176 -0
- package/dist/extraction/secretScanner.js +665 -0
- package/dist/extraction/typedFacts.js +224 -0
- package/dist/extraction/writePathAdjudication.js +110 -0
- package/dist/importing/apply.js +672 -0
- package/dist/importing/bundle-write.js +80 -0
- package/dist/importing/cli.js +408 -0
- package/dist/importing/existing.js +51 -0
- package/dist/importing/facts.js +121 -0
- package/dist/importing/forgetImported.js +79 -0
- package/dist/importing/hosted.js +467 -0
- package/dist/importing/journalLog.js +96 -0
- package/dist/importing/markdown.js +98 -0
- package/dist/importing/plan.js +201 -0
- package/dist/importing/secretScreen.js +113 -0
- package/dist/importing/serialize.js +188 -0
- package/dist/importing/source.js +463 -0
- package/dist/importing/types.js +6 -0
- package/dist/indexer/chunker.js +319 -0
- package/dist/indexer/dailyEntries.js +63 -0
- package/dist/indexer/hasher.js +10 -0
- package/dist/indexer/laneRuns.js +48 -0
- package/dist/indexer/o200k.js +481 -0
- package/dist/indexer/o200k.json +8 -0
- package/dist/indexer/outsideRoot.js +77 -0
- package/dist/indexer/parser.js +170 -0
- package/dist/indexer/pipeline.js +988 -0
- package/dist/indexer/staleness.js +201 -0
- package/dist/indexer/tokenizer.js +77 -0
- package/dist/indexer/walker.js +170 -0
- package/dist/mcp/hive.js +88 -0
- package/dist/mcp/requestErrors.js +47 -0
- package/dist/mcp/server.js +426 -0
- package/dist/memory/agentLease.js +43 -0
- package/dist/memory/confinedWrite.js +315 -0
- package/dist/memory/controls.js +73 -0
- package/dist/memory/dailyNote.js +265 -0
- package/dist/memory/durableAppend.js +57 -0
- package/dist/memory/factConflicts.js +203 -0
- package/dist/memory/factTerms.js +321 -0
- package/dist/memory/handoff.js +363 -0
- package/dist/memory/hiveNameHeader.cjs +10 -0
- package/dist/memory/inventory.js +55 -0
- package/dist/memory/invisibleText.js +20 -0
- package/dist/memory/lifecycle.js +1991 -0
- package/dist/memory/limits.js +13 -0
- package/dist/memory/requests.js +183 -0
- package/dist/memory/sourceName.js +276 -0
- package/dist/memory/statusFacts.js +61 -0
- package/dist/memory/whatsNew.js +158 -0
- package/dist/memory/writer.js +175 -0
- package/dist/memory/writerConflicts.js +85 -0
- package/dist/memorymap/memoryMap.js +1594 -0
- package/dist/memorymap/replacedJournal.js +58 -0
- package/dist/memorymap/replacements.js +410 -0
- package/dist/mvp/activity.js +86 -0
- package/dist/mvp/agentMemory.js +1800 -0
- package/dist/mvp/answerCheck.js +211 -0
- package/dist/mvp/auth.js +531 -0
- package/dist/mvp/cli.js +75 -0
- package/dist/mvp/client-store.js +462 -0
- package/dist/mvp/compileCache.js +242 -0
- package/dist/mvp/compileSeed.js +33 -0
- package/dist/mvp/config.js +288 -0
- package/dist/mvp/connectionStatus.js +80 -0
- package/dist/mvp/contextDistinct.js +53 -0
- package/dist/mvp/contextProfile.js +87 -0
- package/dist/mvp/contextRelevance.js +413 -0
- package/dist/mvp/directorySurface.js +68 -0
- package/dist/mvp/engine.js +1078 -0
- package/dist/mvp/exportAll.js +374 -0
- package/dist/mvp/factLanePack.js +77 -0
- package/dist/mvp/factMarkers.js +65 -0
- package/dist/mvp/factsLane.js +203 -0
- package/dist/mvp/hiddenSpans.js +585 -0
- package/dist/mvp/history.js +414 -0
- package/dist/mvp/hitDates.js +31 -0
- package/dist/mvp/hitFloor.js +36 -0
- package/dist/mvp/identity.js +18 -0
- package/dist/mvp/idle.js +113 -0
- package/dist/mvp/log.js +118 -0
- package/dist/mvp/memoryDir.js +98 -0
- package/dist/mvp/posthogMcp.js +210 -0
- package/dist/mvp/readScope.js +104 -0
- package/dist/mvp/readUsefulness.js +222 -0
- package/dist/mvp/sanitize.js +48 -0
- package/dist/mvp/schemaRejections.js +91 -0
- package/dist/mvp/scope.js +127 -0
- package/dist/mvp/server.js +2604 -0
- package/dist/mvp/serverCard.js +38 -0
- package/dist/mvp/storedSize.js +139 -0
- package/dist/mvp/telemetry.js +474 -0
- package/dist/net/poolFetch.js +174 -0
- package/dist/recall/actionRecall.js +282 -0
- package/dist/recall/autoRecall.js +447 -0
- package/dist/recall/searchRecall.js +189 -0
- package/dist/retrieval/adviceLines.js +195 -0
- package/dist/retrieval/adviceQuestion.js +44 -0
- package/dist/retrieval/conversationShape.js +44 -0
- package/dist/retrieval/dailyEntryTime.js +55 -0
- package/dist/retrieval/entityLookup.js +162 -0
- package/dist/retrieval/factsFirst.js +188 -0
- package/dist/retrieval/keyword.js +84 -0
- package/dist/retrieval/ledgerEventTime.js +51 -0
- package/dist/retrieval/ledgerLiveness.js +83 -0
- package/dist/retrieval/ledgerPresentation.js +161 -0
- package/dist/retrieval/noteIdentity.js +57 -0
- package/dist/retrieval/orchestrator.js +1171 -0
- package/dist/retrieval/plainWording.js +236 -0
- package/dist/retrieval/queryHelpers.js +142 -0
- package/dist/retrieval/questionSpan.js +475 -0
- package/dist/retrieval/readerEvents.js +690 -0
- package/dist/retrieval/readerLayout.js +422 -0
- package/dist/retrieval/relativeTimeHints.js +113 -0
- package/dist/retrieval/semantic.js +67 -0
- package/dist/router/routerCache.js +106 -0
- package/dist/router/sessionRouter.js +726 -0
- package/dist/toolRefusal.js +120 -0
- package/dist/watcher/watcher.js +77 -0
- package/package.json +118 -4
|
@@ -0,0 +1,422 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.READER_SESSION_CAP = exports.LAYOUT_MIN_TURNS = void 0;
|
|
4
|
+
exports.readerLayoutEnabled = readerLayoutEnabled;
|
|
5
|
+
exports.readerLayoutSkipsAdvice = readerLayoutSkipsAdvice;
|
|
6
|
+
exports.readerLayoutEnabledFor = readerLayoutEnabledFor;
|
|
7
|
+
exports.readerTimelineEnabled = readerTimelineEnabled;
|
|
8
|
+
exports.isCurrentValueQuestion = isCurrentValueQuestion;
|
|
9
|
+
exports.currentValueTimeline = currentValueTimeline;
|
|
10
|
+
exports.isConversation = isConversation;
|
|
11
|
+
exports.conversationNoteMatcher = conversationNoteMatcher;
|
|
12
|
+
exports.readerSessionCap = readerSessionCap;
|
|
13
|
+
exports.readerBlockOrder = readerBlockOrder;
|
|
14
|
+
exports.isHeaderOnly = isHeaderOnly;
|
|
15
|
+
exports.layoutHitsForReader = layoutHitsForReader;
|
|
16
|
+
exports.layoutPage = layoutPage;
|
|
17
|
+
exports.readerLayoutPage = readerLayoutPage;
|
|
18
|
+
exports.readerLayoutOf = readerLayoutOf;
|
|
19
|
+
const relativeTimeHints_1 = require("./relativeTimeHints");
|
|
20
|
+
const orchestrator_1 = require("./orchestrator");
|
|
21
|
+
const questionSpan_1 = require("./questionSpan");
|
|
22
|
+
const readerEvents_1 = require("./readerEvents");
|
|
23
|
+
const conversationShape_1 = require("./conversationShape");
|
|
24
|
+
const adviceQuestion_1 = require("./adviceQuestion");
|
|
25
|
+
/**
|
|
26
|
+
* How a search reply is laid out for the model that reads it. Retrieval picks the window and orders it; this only
|
|
27
|
+
* changes how what it picked is handed over, and it never cuts or rewrites a word of an excerpt it shows:
|
|
28
|
+
*
|
|
29
|
+
* - one hit per session: the chunks of a note on one date, which the newest-first layout left apart (a session's header,
|
|
30
|
+
* which carries its date, could come after the turns it dates, and turns of other sessions in between), come as one hit,
|
|
31
|
+
* in the note's own order, so every turn sits under the date of its session;
|
|
32
|
+
* - a session whose only chunk is its header (`# Session x` and a `Timestamp:` line) is no hit: it dated nothing the reader
|
|
33
|
+
* can see and took a place in the reply (a third of the hits of the 2026-10-16 LongMemEval-S run);
|
|
34
|
+
* - the blocks stand where the page had them (newest first by default). RENOR_READER_ORDER=relevance puts them best match
|
|
35
|
+
* first instead (a block's score is its best chunk's, ties newest first) and the reply says so (`rankedBy: score`,
|
|
36
|
+
* readerLayoutPage; only a page whose ordinary notes are all conversations). It is off by default: on RenorBench v2 it puts a
|
|
37
|
+
* superseded value ahead of the current one (tests/eval/renorbench-v2.test.ts);
|
|
38
|
+
* - RENOR_READER_SESSION_CAP=N lets a session show at most N chunks besides its header, except that every chunk the query
|
|
39
|
+
* matched stays: the cap only stops the statements the session expansion added. Off by default (0): on RenorBench v2 a cap
|
|
40
|
+
* of 10 drops the remark a question asks about, which only the expansion delivers;
|
|
41
|
+
* - a session that reaches the reader without its header (the date line is its first chunk) opens with
|
|
42
|
+
* `[Session date: <date>]`;
|
|
43
|
+
* - each time phrase that names one point in time ("three weeks ago", "last Thursday") is resolved to its date below
|
|
44
|
+
* the excerpt (relativeTimeHints.ts).
|
|
45
|
+
*
|
|
46
|
+
* Whole chunks leave (a header stub, a statement over the cap); no chunk is cut or reworded.
|
|
47
|
+
*
|
|
48
|
+
* Only notes shaped like a conversation (conversationShape.ts) are laid out: a session is one conversation, and merging
|
|
49
|
+
* the chunks of a document or the entries of a daily note would break "a hit is one memory".
|
|
50
|
+
*
|
|
51
|
+
* Measured on LongMemEval-S dev100 with `gemma4:31b` as reader and judge: see docs/evals/results/READER-LAYOUT-20261014.md.
|
|
52
|
+
* Not applied to get_context (its budget cuts whole items by relevance) or to any hit that is no chunk of an ordinary
|
|
53
|
+
* note: a remembered fact, a ledger event, a correction, the evidence or digest table. Kill switch: RENOR_READER_LAYOUT=0.
|
|
54
|
+
*
|
|
55
|
+
* On by default, and only on top of this layout (`RENOR_READER_EVENTS=0` switches it off; then a layout-on page is
|
|
56
|
+
* byte-identical): when the question asks for the order or the date of something the user did, one line per matching
|
|
57
|
+
* statement of the user's, dated and oldest first, is placed above the excerpts (readerEvents.ts). A duration question and
|
|
58
|
+
* any other question are left untouched.
|
|
59
|
+
*
|
|
60
|
+
* Opt-in, and only on top of this layout (`RENOR_READER_TIMELINE=1`; default off, so a layout-on page is byte-identical):
|
|
61
|
+
* when the question asks what a value is now, one timeline of the user turns that state it is placed above the excerpts.
|
|
62
|
+
* Oldest first, the latest line marked, earlier lines kept. It quotes those turns and does not cut or reword an excerpt,
|
|
63
|
+
* invent a value, or do arithmetic on a session header (a header date orders two different calendar days only; a shared
|
|
64
|
+
* day is not a clock). A duration question or an order question is left untouched.
|
|
65
|
+
*/
|
|
66
|
+
function readerLayoutEnabled() {
|
|
67
|
+
const v = (process.env.RENOR_READER_LAYOUT ?? '').trim().toLowerCase();
|
|
68
|
+
return !(v === '0' || v === 'false' || v === 'no' || v === 'off');
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Default off. `1`, `true`, `yes` or `on` leaves a query that asks to be advised (adviceQuestion.ts: a recommendation, a
|
|
72
|
+
* suggestion, tips, ideas, an opinion) without the layout, so its page is the one RENOR_READER_LAYOUT=0 gives. Every other query,
|
|
73
|
+
* and every query with this unset, is laid out exactly as before.
|
|
74
|
+
*/
|
|
75
|
+
function readerLayoutSkipsAdvice() {
|
|
76
|
+
const v = (process.env.RENOR_READER_LAYOUT_SKIP_ADVICE ?? '').trim().toLowerCase();
|
|
77
|
+
return v === '1' || v === 'true' || v === 'yes' || v === 'on';
|
|
78
|
+
}
|
|
79
|
+
/** Is the page of this query laid out? The kill switch decides first, then the advice exception. */
|
|
80
|
+
function readerLayoutEnabledFor(query) {
|
|
81
|
+
return readerLayoutEnabled() && !(readerLayoutSkipsAdvice() && (0, adviceQuestion_1.asksForAdvice)(query));
|
|
82
|
+
}
|
|
83
|
+
/** Default off. `1` / `true` / `yes` / `on` places a current-value timeline above a layout-on page. */
|
|
84
|
+
function readerTimelineEnabled(env = process.env) {
|
|
85
|
+
const v = (env.RENOR_READER_TIMELINE ?? '').trim().toLowerCase();
|
|
86
|
+
return v === '1' || v === 'true' || v === 'yes' || v === 'on';
|
|
87
|
+
}
|
|
88
|
+
// A duration or an order stays byte-identical. "Ago" and "last Tuesday" are a point in the past, not a changed value.
|
|
89
|
+
const DURATION_QUESTION = /\b(?:how (?:many|much) (?:days|weeks|months|years)|how long|how much time|how old)\b|\b(?:days|weeks|months|years)\b[^?]{0,24}\b(?:ago|passed)\b|\bago\b|\bpassed between\b|\blast (?:week|month|year|weekend|monday|tuesday|wednesday|thursday|friday|saturday|sunday)\b/i;
|
|
90
|
+
const ORDER_QUESTION = /\border\b|\b(?:first to last|earliest to latest|happened first|came first)\b|\b(?:which|who)\b[^?]{0,80}\bfirst\b/i;
|
|
91
|
+
const CURRENT_VALUE_QUESTION = /\b(?:now|currently|current|most recent(?:ly)?|latest|so far|previous(?:ly)?|updated|initially|what changed|used to|how many|how often|how much|what|where|did i|do i|am i|is my|have i)\b/i;
|
|
92
|
+
/** A knowledge-update-shaped question: what the value is now, or what it was before it changed. Not a duration or an order. */
|
|
93
|
+
function isCurrentValueQuestion(query) {
|
|
94
|
+
const q = (query.split('\n').pop() ?? query).trim();
|
|
95
|
+
if (q === '' || DURATION_QUESTION.test(q) || ORDER_QUESTION.test(q))
|
|
96
|
+
return false;
|
|
97
|
+
return CURRENT_VALUE_QUESTION.test(q);
|
|
98
|
+
}
|
|
99
|
+
const QUOTE_STOP = new Set([
|
|
100
|
+
'what', 'where', 'when', 'which', 'who', 'whom', 'how', 'many', 'much', 'often', 'does', 'did', 'do', 'have', 'has',
|
|
101
|
+
'had', 'was', 'were', 'been', 'being', 'this', 'that', 'these', 'those', 'with', 'from', 'your', 'about', 'into',
|
|
102
|
+
'over', 'under', 'after', 'before', 'between', 'since', 'until', 'while', 'just', 'still', 'also', 'than', 'then',
|
|
103
|
+
'them', 'they', 'their', 'there', 'here', 'would', 'could', 'should', 'will', 'shall', 'can', 'may', 'might', 'must',
|
|
104
|
+
'some', 'any', 'more', 'most', 'other', 'only', 'same', 'very', 'really', 'around', 'time', 'times', 'when',
|
|
105
|
+
]);
|
|
106
|
+
function contentTokens(question) {
|
|
107
|
+
const words = question.toLowerCase().match(/[a-z0-9$][a-z0-9$'-]{3,}/g) ?? [];
|
|
108
|
+
return new Set(words.filter(word => !QUOTE_STOP.has(word) && !/^\d+$/.test(word)));
|
|
109
|
+
}
|
|
110
|
+
function sameWord(a, b) {
|
|
111
|
+
if (a === b)
|
|
112
|
+
return true;
|
|
113
|
+
const [longer, shorter] = a.length >= b.length ? [a, b] : [b, a];
|
|
114
|
+
return longer === `${shorter}s` || longer === `${shorter}es`;
|
|
115
|
+
}
|
|
116
|
+
function sharesToken(sentence, tokens) {
|
|
117
|
+
const words = sentence.toLowerCase().match(/[a-z0-9$][a-z0-9$'-]{3,}/g) ?? [];
|
|
118
|
+
return words.some(word => {
|
|
119
|
+
for (const token of tokens)
|
|
120
|
+
if (sameWord(word, token))
|
|
121
|
+
return true;
|
|
122
|
+
return false;
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
function headingOf(line) {
|
|
126
|
+
const match = /^#{1,6}\s+(.*)$/.exec(line.trim());
|
|
127
|
+
return match ? match[1].trim() : null;
|
|
128
|
+
}
|
|
129
|
+
function headingWords(heading) {
|
|
130
|
+
return heading.toLowerCase().replace(/[\p{N}\p{P}\p{S}\p{Z}]+/gu, ' ').trim().split(' ').filter(Boolean);
|
|
131
|
+
}
|
|
132
|
+
function isUserTurnHeading(line) {
|
|
133
|
+
const heading = headingOf(line);
|
|
134
|
+
if (heading === null || !(0, conversationShape_1.isTurnLabel)(heading))
|
|
135
|
+
return false;
|
|
136
|
+
const words = headingWords(heading);
|
|
137
|
+
return words.includes('user') || words.includes('human');
|
|
138
|
+
}
|
|
139
|
+
function isTurnHeading(line) {
|
|
140
|
+
const heading = headingOf(line);
|
|
141
|
+
return heading !== null && (0, conversationShape_1.isTurnLabel)(heading);
|
|
142
|
+
}
|
|
143
|
+
/** The user's own words in one laid-out hit, one string per user turn. Assistant turns are not quoted. */
|
|
144
|
+
function userTurnBodies(text) {
|
|
145
|
+
const lines = text.split(/\r?\n/);
|
|
146
|
+
const bodies = [];
|
|
147
|
+
for (let i = 0; i < lines.length; i++) {
|
|
148
|
+
if (!isUserTurnHeading(lines[i]))
|
|
149
|
+
continue;
|
|
150
|
+
const buf = [];
|
|
151
|
+
for (let j = i + 1; j < lines.length; j++) {
|
|
152
|
+
const trimmed = lines[j].trim();
|
|
153
|
+
if (isTurnHeading(lines[j]))
|
|
154
|
+
break;
|
|
155
|
+
if (trimmed.startsWith('[Counted from this session\'s date') || trimmed.startsWith('[Session date:'))
|
|
156
|
+
continue;
|
|
157
|
+
buf.push(lines[j]);
|
|
158
|
+
}
|
|
159
|
+
const body = buf.join(' ').replace(/\s+/g, ' ').trim();
|
|
160
|
+
if (body !== '')
|
|
161
|
+
bodies.push(body);
|
|
162
|
+
}
|
|
163
|
+
return bodies;
|
|
164
|
+
}
|
|
165
|
+
function quoteSentences(body, tokens) {
|
|
166
|
+
const sentences = body.split(/(?<=[.!?])\s+/).map(sentence => sentence.trim()).filter(sentence => sentence !== '');
|
|
167
|
+
const kept = (sentences.length > 0 ? sentences : [body]).filter(sentence => sharesToken(sentence, tokens));
|
|
168
|
+
if (kept.length === 0)
|
|
169
|
+
return null;
|
|
170
|
+
const best = kept.slice().sort((a, b) => Number(sharesToken(b, tokens)) - Number(sharesToken(a, tokens)) || b.length - a.length)[0];
|
|
171
|
+
const text = kept.length === 1 ? kept[0] : [best, ...kept.filter(sentence => sentence !== best)].slice(0, 2).join(' ');
|
|
172
|
+
if (text.length <= 500)
|
|
173
|
+
return text;
|
|
174
|
+
const lower = best.toLowerCase();
|
|
175
|
+
let at = -1;
|
|
176
|
+
for (const token of tokens) {
|
|
177
|
+
const found = lower.indexOf(token);
|
|
178
|
+
if (found >= 0 && (at < 0 || found < at))
|
|
179
|
+
at = found;
|
|
180
|
+
}
|
|
181
|
+
if (at < 0)
|
|
182
|
+
return best.slice(0, 500).trim();
|
|
183
|
+
const start = Math.max(0, at - 180);
|
|
184
|
+
return best.slice(start, Math.min(best.length, start + 500)).trim();
|
|
185
|
+
}
|
|
186
|
+
/** Calendar day only. The clock on a shared day is not an order: those stamps are often the question's own day. */
|
|
187
|
+
function calendarDay(sessionDate) {
|
|
188
|
+
const match = /^(\d{4})[/-](\d{2})[/-](\d{2})/.exec(sessionDate ?? '');
|
|
189
|
+
return match ? `${match[1]}-${match[2]}-${match[3]}` : null;
|
|
190
|
+
}
|
|
191
|
+
function sessionIndex(notePath) {
|
|
192
|
+
const base = notePath.split('/').pop() ?? notePath;
|
|
193
|
+
const match = /_(\d+)(?:-|\.|$)/.exec(base);
|
|
194
|
+
return match ? Number(match[1]) : null;
|
|
195
|
+
}
|
|
196
|
+
/**
|
|
197
|
+
* The timeline line, or null when this question is not a current-value question or no user turn states one.
|
|
198
|
+
* Nothing here is invented: every quote is a substring of a user turn that is already in `hits`.
|
|
199
|
+
*/
|
|
200
|
+
function currentValueTimeline(hits, query) {
|
|
201
|
+
if (!isCurrentValueQuestion(query))
|
|
202
|
+
return null;
|
|
203
|
+
const tokens = contentTokens(query.split('\n').pop() ?? query);
|
|
204
|
+
if (tokens.size === 0)
|
|
205
|
+
return null;
|
|
206
|
+
const stated = [];
|
|
207
|
+
const seen = new Set();
|
|
208
|
+
hits.forEach((hit, appearance) => {
|
|
209
|
+
const day = calendarDay(hit.session_date);
|
|
210
|
+
const index = sessionIndex(hit.notePath);
|
|
211
|
+
for (const body of userTurnBodies(hit.text)) {
|
|
212
|
+
const text = quoteSentences(body, tokens);
|
|
213
|
+
if (text === null)
|
|
214
|
+
continue;
|
|
215
|
+
// One statement per note: the same words in a later session are a later statement (A, B, A ends on A), not a repeat.
|
|
216
|
+
const key = `${hit.notePath}\u0000${text.toLowerCase()}`;
|
|
217
|
+
if (seen.has(key))
|
|
218
|
+
continue;
|
|
219
|
+
seen.add(key);
|
|
220
|
+
stated.push({ day, index, chunk: hit.chunkId ?? 0, appearance, text });
|
|
221
|
+
}
|
|
222
|
+
});
|
|
223
|
+
if (stated.length === 0)
|
|
224
|
+
return null;
|
|
225
|
+
stated.sort((a, b) => {
|
|
226
|
+
if (a.day !== null && b.day !== null && a.day !== b.day)
|
|
227
|
+
return a.day < b.day ? -1 : 1;
|
|
228
|
+
if (a.index !== null && b.index !== null && a.index !== b.index)
|
|
229
|
+
return a.index - b.index;
|
|
230
|
+
if (a.chunk !== b.chunk)
|
|
231
|
+
return a.chunk - b.chunk;
|
|
232
|
+
return a.appearance - b.appearance;
|
|
233
|
+
});
|
|
234
|
+
const lines = stated.map((value, i) => (i === stated.length - 1 ? `- latest: "${value.text}"` : `- "${value.text}"`));
|
|
235
|
+
return ['Stated by the user, oldest first. Earlier values stay. The last line is the latest.', ...lines].join('\n');
|
|
236
|
+
}
|
|
237
|
+
function withCurrentValueTimeline(hits, query) {
|
|
238
|
+
if (!readerTimelineEnabled() || query === undefined || query.trim() === '')
|
|
239
|
+
return hits;
|
|
240
|
+
const text = currentValueTimeline(hits, query);
|
|
241
|
+
if (text === null)
|
|
242
|
+
return hits;
|
|
243
|
+
return [{ notePath: '', text, score: 0, reason: 'timeline' }, ...hits];
|
|
244
|
+
}
|
|
245
|
+
/**
|
|
246
|
+
* A note of turns: at least `LAYOUT_MIN_TURNS` of its distinct heading paths end in a speaker or turn label (conversationShape.ts
|
|
247
|
+
* isTurnLabel) and those are at least half of its paths. One exchange, a question and its answer, is a conversation here: the
|
|
248
|
+
* session expansion's own rule asks for four turns because it cuts statements, and this one only joins what a session already is.
|
|
249
|
+
*/
|
|
250
|
+
exports.LAYOUT_MIN_TURNS = 2;
|
|
251
|
+
function isConversation(headingPaths) {
|
|
252
|
+
const distinct = new Set(headingPaths);
|
|
253
|
+
let turns = 0;
|
|
254
|
+
for (const headingPath of distinct)
|
|
255
|
+
if ((0, conversationShape_1.isTurnLabel)(headingPath.split(' > ').pop() ?? ''))
|
|
256
|
+
turns += 1;
|
|
257
|
+
return turns >= exports.LAYOUT_MIN_TURNS && turns * 2 >= distinct.size;
|
|
258
|
+
}
|
|
259
|
+
/** Is this stored note path a conversation? Told from the headings of its chunks, once per note. */
|
|
260
|
+
function conversationNoteMatcher(db) {
|
|
261
|
+
const select = db.prepare('SELECT heading_path AS headingPath FROM chunks WHERE note_path = ?');
|
|
262
|
+
const known = new Map();
|
|
263
|
+
return storedNotePath => {
|
|
264
|
+
let shaped = known.get(storedNotePath);
|
|
265
|
+
if (shaped === undefined) {
|
|
266
|
+
shaped = isConversation(select.all(storedNotePath).map(row => row.headingPath));
|
|
267
|
+
known.set(storedNotePath, shaped);
|
|
268
|
+
}
|
|
269
|
+
return shaped;
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
function isNoteChunk(hit, isLedger) {
|
|
273
|
+
return (hit.chunkId ?? 0) > 0 && hit.notePath !== '' && !isLedger(hit.notePath) && hit.author === undefined && hit.replaced === undefined;
|
|
274
|
+
}
|
|
275
|
+
/** The most chunks of one session's block (its header aside); RENOR_READER_SESSION_CAP, 0 (the default) is no cap. */
|
|
276
|
+
exports.READER_SESSION_CAP = 0;
|
|
277
|
+
function readerSessionCap(env = process.env) {
|
|
278
|
+
const raw = (env.RENOR_READER_SESSION_CAP ?? '').trim();
|
|
279
|
+
if (raw === '')
|
|
280
|
+
return exports.READER_SESSION_CAP;
|
|
281
|
+
const n = Number(raw);
|
|
282
|
+
return Number.isInteger(n) && n >= 0 ? n : exports.READER_SESSION_CAP;
|
|
283
|
+
}
|
|
284
|
+
/** Blocks keep the page's own (newest-first) order; RENOR_READER_ORDER=relevance puts them best match first. */
|
|
285
|
+
function readerBlockOrder(env = process.env) {
|
|
286
|
+
return (env.RENOR_READER_ORDER ?? '').trim().toLowerCase() === 'relevance' ? 'relevance' : 'date';
|
|
287
|
+
}
|
|
288
|
+
/**
|
|
289
|
+
* A chunk that is only a session's header: headings and `Timestamp:` lines, no turn. It dates the turns under it and says
|
|
290
|
+
* nothing itself; on its own it is a stub that takes a place in the reply (a third of the hits of the 2026-10-16 run).
|
|
291
|
+
*/
|
|
292
|
+
function isHeaderOnly(text) {
|
|
293
|
+
const lines = text.split(/\r?\n/).map(line => line.trim()).filter(line => line !== '' && !/^#{1,6}\s/.test(line));
|
|
294
|
+
return lines.length > 0 && lines.every(line => /^Timestamp:/i.test(line));
|
|
295
|
+
}
|
|
296
|
+
/** A hit of an ordinary note (a chunk the layout could place, a daily entry, a document paragraph): not a fact, a ledger event, a correction or the evidence and digest tables (`_renor/…`). */
|
|
297
|
+
function isOrdinaryNoteHit(hit, isLedger) {
|
|
298
|
+
return hit.notePath !== '' && !hit.notePath.startsWith('_renor/') && !isLedger(hit.notePath) && hit.author === undefined && hit.replaced === undefined;
|
|
299
|
+
}
|
|
300
|
+
function layoutHitsForReader(hits, opts) {
|
|
301
|
+
return layoutPage(hits, opts).hits;
|
|
302
|
+
}
|
|
303
|
+
/**
|
|
304
|
+
* `byRelevance` is true when the blocks were placed best match first, which the reply then says (`rankedBy: score`). It is
|
|
305
|
+
* done only to a page whose every chunk of an ordinary note is a conversation's: a daily note's entries or a document's
|
|
306
|
+
* paragraphs on the same page keep the order the page gave them, and a page that is half one order and half the other must
|
|
307
|
+
* not claim either key.
|
|
308
|
+
*/
|
|
309
|
+
function layoutPage(hits, opts) {
|
|
310
|
+
const chunk = (hit) => isNoteChunk(hit, opts.isLedger) && (opts.isConversation?.(hit.notePath) ?? true);
|
|
311
|
+
const cap = opts.sessionCap ?? readerSessionCap();
|
|
312
|
+
const order = opts.order ?? readerBlockOrder();
|
|
313
|
+
const laid = hits.map(hit => (chunk(hit) ? { ...hit, text: (0, relativeTimeHints_1.withRelativeTimeHints)(hit.text, hit.session_date) } : hit));
|
|
314
|
+
// One block per conversation on one date: every chunk of it under that date, wherever the page had put it.
|
|
315
|
+
const keyOf = (hit) => (chunk(hit) ? `${hit.notePath}\0${hit.session_date ?? ''}` : null);
|
|
316
|
+
const blocks = new Map();
|
|
317
|
+
laid.forEach((hit, index) => {
|
|
318
|
+
const key = keyOf(hit);
|
|
319
|
+
if (key === null)
|
|
320
|
+
return;
|
|
321
|
+
const block = blocks.get(key) ?? { header: [], body: [], first: index };
|
|
322
|
+
(isHeaderOnly(hit.text) ? block.header : block.body).push(hit);
|
|
323
|
+
blocks.set(key, block);
|
|
324
|
+
});
|
|
325
|
+
const made = new Map();
|
|
326
|
+
for (const [key, block] of blocks)
|
|
327
|
+
made.set(key, blockHit(block, cap));
|
|
328
|
+
// The blocks stand in the places the page gave its chunks; anything else keeps its place.
|
|
329
|
+
const slots = new Set(laid.flatMap((hit, index) => (keyOf(hit) !== null && blocks.get(keyOf(hit)).first === index ? [index] : [])));
|
|
330
|
+
let placed = [...slots].map(index => keyOf(laid[index])).filter(key => made.get(key) !== null);
|
|
331
|
+
const byRelevance = order === 'relevance' && placed.length > 0 && hits.every(hit => !isOrdinaryNoteHit(hit, opts.isLedger) || chunk(hit));
|
|
332
|
+
if (byRelevance) {
|
|
333
|
+
const at = new Map(placed.map((key, i) => [key, i]));
|
|
334
|
+
placed = [...placed].sort((a, b) => made.get(b).score - made.get(a).score
|
|
335
|
+
|| ((0, orchestrator_1.sessionDateMs)(made.get(b).session_date) || 0) - ((0, orchestrator_1.sessionDateMs)(made.get(a).session_date) || 0)
|
|
336
|
+
|| at.get(a) - at.get(b));
|
|
337
|
+
}
|
|
338
|
+
const out = [];
|
|
339
|
+
let next = 0;
|
|
340
|
+
laid.forEach((hit, index) => {
|
|
341
|
+
const key = keyOf(hit);
|
|
342
|
+
if (key === null)
|
|
343
|
+
out.push(hit);
|
|
344
|
+
else if (slots.has(index) && made.get(key) !== null)
|
|
345
|
+
out.push(made.get(placed[next++]));
|
|
346
|
+
});
|
|
347
|
+
const dated = out.map(hit => (chunk(hit) && hit.session_date && !/Timestamp:/i.test(hit.text)
|
|
348
|
+
? { ...hit, text: `[Session date: ${hit.session_date}]\n${hit.text}` }
|
|
349
|
+
: hit));
|
|
350
|
+
// RENOR_READER_DURATION (default on; `0` switches it off): one line of dates the user wrote, first, for a "how many days between" question (questionSpan.ts).
|
|
351
|
+
// It reads the chunks as they were found: joined, a turn's continuation would sit under the speaker before it.
|
|
352
|
+
// RENOR_READER_EVENTS (default on; `0` switches it off) adds event lines for an order or a date question; its gate leaves a duration question alone, so at most one of the two lines is added.
|
|
353
|
+
// RENOR_READER_TIMELINE (default off): a current-value timeline for a "what is it now" question; a duration or an order question is left alone.
|
|
354
|
+
// Each such line comes first, outside the order rankedBy names, as the evidence hit does. The event lines too read the
|
|
355
|
+
// conversations as the page found them: a session that came as its header alone is no hit, but its turns are still read.
|
|
356
|
+
return {
|
|
357
|
+
hits: withCurrentValueTimeline(withEventLines((0, questionSpan_1.withQuestionSpan)(dated, opts.query, { source: hits }), hits.filter(chunk), opts), opts.query),
|
|
358
|
+
byRelevance,
|
|
359
|
+
};
|
|
360
|
+
}
|
|
361
|
+
/** The event lines, as the first hit (reason "events"), when the flag is on and the question asks for an order or a date. `sources` are the conversation chunks they read. */
|
|
362
|
+
function withEventLines(hits, sources, opts) {
|
|
363
|
+
if (!(0, readerEvents_1.readerEventsEnabled)() || opts.query === undefined || opts.turnsOf === undefined || opts.query.trim() === '')
|
|
364
|
+
return hits;
|
|
365
|
+
const text = (0, readerEvents_1.eventLines)(sources, opts.query, opts.turnsOf, { scoreTurns: opts.scoreTurns });
|
|
366
|
+
if (text === null)
|
|
367
|
+
return hits;
|
|
368
|
+
return [{ notePath: '', text, score: 0, reason: 'events' }, ...hits];
|
|
369
|
+
}
|
|
370
|
+
/**
|
|
371
|
+
* One conversation's hit: its header first when it came, then its turns in the conversation's own order, with the best
|
|
372
|
+
* score and reason of its chunks. A session that came as a header and nothing else is no hit. Over the cap, every chunk the
|
|
373
|
+
* query matched stays and the statements the session expansion added are taken in the order they came, as many as the cap leaves room for.
|
|
374
|
+
*/
|
|
375
|
+
function blockHit(block, cap) {
|
|
376
|
+
if (block.body.length === 0)
|
|
377
|
+
return null;
|
|
378
|
+
const byPlace = (a, b) => (a.chunkId ?? 0) - (b.chunkId ?? 0);
|
|
379
|
+
let body = block.body;
|
|
380
|
+
if (cap > 0 && body.length > cap) {
|
|
381
|
+
// What the query matched always stays; the cap only limits the statements the session expansion added.
|
|
382
|
+
const expanded = body.filter(h => h.reason === orchestrator_1.SESSION_EXPAND_REASON);
|
|
383
|
+
const room = Math.max(0, cap - (body.length - expanded.length));
|
|
384
|
+
const kept = new Set([...body.filter(h => h.reason !== orchestrator_1.SESSION_EXPAND_REASON), ...expanded.slice(0, room)]);
|
|
385
|
+
body = body.filter(h => kept.has(h));
|
|
386
|
+
}
|
|
387
|
+
const run = [...block.header, ...body].sort(byPlace);
|
|
388
|
+
if (run.length === 1)
|
|
389
|
+
return run[0];
|
|
390
|
+
const best = run.reduce((a, b) => (b.score > a.score ? b : a));
|
|
391
|
+
const similarities = run.flatMap(h => (typeof h.similarity === 'number' ? [h.similarity] : []));
|
|
392
|
+
return {
|
|
393
|
+
...run[0],
|
|
394
|
+
text: run.map(h => h.text).join('\n\n'),
|
|
395
|
+
score: best.score,
|
|
396
|
+
reason: best.reason,
|
|
397
|
+
...(similarities.length > 0 ? { similarity: Math.max(...similarities) } : {}),
|
|
398
|
+
};
|
|
399
|
+
}
|
|
400
|
+
/**
|
|
401
|
+
* The page search_memory shows, laid out for its reader when RENOR_READER_LAYOUT is on (the chunk ids the layout needed
|
|
402
|
+
* leave with it); otherwise the page as it was. `rankedBy` is the key the returned page is in order of. `enabled` is the
|
|
403
|
+
* answer for this query (readerLayoutEnabledFor).
|
|
404
|
+
*/
|
|
405
|
+
function readerLayoutPage(rankedBy, hits, isLedger, events = {}, enabled = readerLayoutEnabled()) {
|
|
406
|
+
if (!enabled)
|
|
407
|
+
return { hits, rankedBy };
|
|
408
|
+
// A bare string is the question alone (the duration line's form); the object also carries what the event lines read.
|
|
409
|
+
const input = typeof events === 'string' ? { query: events } : events;
|
|
410
|
+
const laid = layoutPage(hits, { rankedBy, isLedger, ...input });
|
|
411
|
+
return {
|
|
412
|
+
hits: laid.hits.map(hit => {
|
|
413
|
+
const { chunkId: _chunkId, ...rest } = hit;
|
|
414
|
+
return rest;
|
|
415
|
+
}),
|
|
416
|
+
rankedBy: laid.byRelevance ? 'score' : rankedBy,
|
|
417
|
+
};
|
|
418
|
+
}
|
|
419
|
+
/** The hits of readerLayoutPage alone. */
|
|
420
|
+
function readerLayoutOf(rankedBy, hits, isLedger, events = {}, enabled = readerLayoutEnabled()) {
|
|
421
|
+
return readerLayoutPage(rankedBy, hits, isLedger, events, enabled).hits;
|
|
422
|
+
}
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.formatDay = formatDay;
|
|
4
|
+
exports.relativeTimePoints = relativeTimePoints;
|
|
5
|
+
exports.relativeTimeHints = relativeTimeHints;
|
|
6
|
+
exports.withRelativeTimeHints = withRelativeTimeHints;
|
|
7
|
+
const orchestrator_1 = require("./orchestrator");
|
|
8
|
+
/**
|
|
9
|
+
* Says, below an excerpt, which calendar date each relative time phrase in it points to, counted from the excerpt's own
|
|
10
|
+
* session date: "three weeks ago" in a session of 2023/05/28 is 2023/05/07. A reader that has to do that arithmetic,
|
|
11
|
+
* then the arithmetic of the question on top, often answers "I don't know" with both dates in front of it; the hint
|
|
12
|
+
* does the first step and leaves the stored text, and the question's own arithmetic, alone.
|
|
13
|
+
*
|
|
14
|
+
* English only, and only phrases that name one point in time (a count of days, weeks, months or years ago; yesterday,
|
|
15
|
+
* today; last <weekday>, week, month or year). Nothing is guessed: a phrase this does not match gets no hint, and a hit
|
|
16
|
+
* without a parseable date gets none.
|
|
17
|
+
*/
|
|
18
|
+
const NUMBER_WORDS = {
|
|
19
|
+
a: 1, an: 1, one: 1, two: 2, three: 3, four: 4, five: 5, six: 6, seven: 7, eight: 8, nine: 9, ten: 10, eleven: 11, twelve: 12,
|
|
20
|
+
};
|
|
21
|
+
const WEEKDAYS = ['sunday', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday'];
|
|
22
|
+
const DAY_NAMES = ['Sun', 'Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat'];
|
|
23
|
+
const DAY_MS = 86_400_000;
|
|
24
|
+
const AGO = new RegExp(`\\b(${Object.keys(NUMBER_WORDS).join('|')}|\\d{1,3})\\s+(day|week|month|year)s?\\s+ago\\b`, 'gi');
|
|
25
|
+
const SIMPLE = /\b(yesterday|today|last\s+(?:week|month|year|sunday|monday|tuesday|wednesday|thursday|friday|saturday))\b/gi;
|
|
26
|
+
function pad(n) {
|
|
27
|
+
return String(n).padStart(2, '0');
|
|
28
|
+
}
|
|
29
|
+
function addMonths(date, months) {
|
|
30
|
+
const out = new Date(date.getTime());
|
|
31
|
+
const day = out.getUTCDate();
|
|
32
|
+
out.setUTCDate(1);
|
|
33
|
+
out.setUTCMonth(out.getUTCMonth() + months);
|
|
34
|
+
const last = new Date(Date.UTC(out.getUTCFullYear(), out.getUTCMonth() + 1, 0)).getUTCDate();
|
|
35
|
+
out.setUTCDate(Math.min(day, last));
|
|
36
|
+
return out;
|
|
37
|
+
}
|
|
38
|
+
/** A day as the hit's own date style writes it: `2023/05/07 (Sun)` for the benchmark's slash dates, `2023-05-07` otherwise. */
|
|
39
|
+
function formatDay(date, slashStyle) {
|
|
40
|
+
const ymd = slashStyle ? `${date.getUTCFullYear()}/${pad(date.getUTCMonth() + 1)}/${pad(date.getUTCDate())}` : `${date.getUTCFullYear()}-${pad(date.getUTCMonth() + 1)}-${pad(date.getUTCDate())}`;
|
|
41
|
+
return slashStyle ? `${ymd} (${DAY_NAMES[date.getUTCDay()]})` : ymd;
|
|
42
|
+
}
|
|
43
|
+
/** The phrases of one excerpt that name a point in time, in the order they appear, each phrase once. */
|
|
44
|
+
function relativeTimePoints(text, sessionDate) {
|
|
45
|
+
const ms = (0, orchestrator_1.sessionDateMs)(sessionDate);
|
|
46
|
+
if (!Number.isFinite(ms) || sessionDate === undefined)
|
|
47
|
+
return [];
|
|
48
|
+
const slashStyle = /^\d{4}\//.test(sessionDate);
|
|
49
|
+
const base = new Date(Math.floor(ms / DAY_MS) * DAY_MS);
|
|
50
|
+
const found = [];
|
|
51
|
+
for (const m of text.matchAll(AGO)) {
|
|
52
|
+
const raw = m[1].toLowerCase();
|
|
53
|
+
const n = raw in NUMBER_WORDS ? NUMBER_WORDS[raw] : Number(raw);
|
|
54
|
+
const unit = m[2].toLowerCase();
|
|
55
|
+
const date = unit === 'day' ? new Date(base.getTime() - n * DAY_MS)
|
|
56
|
+
: unit === 'week' ? new Date(base.getTime() - 7 * n * DAY_MS)
|
|
57
|
+
: unit === 'month' ? addMonths(base, -n)
|
|
58
|
+
: addMonths(base, -12 * n);
|
|
59
|
+
found.push({ at: m.index ?? 0, phrase: m[0], date, value: formatDay(date, slashStyle), grain: 'day' });
|
|
60
|
+
}
|
|
61
|
+
for (const m of text.matchAll(SIMPLE)) {
|
|
62
|
+
const phrase = m[1].toLowerCase().replace(/\s+/g, ' ');
|
|
63
|
+
let date;
|
|
64
|
+
let value;
|
|
65
|
+
let grain = 'day';
|
|
66
|
+
if (phrase === 'today') {
|
|
67
|
+
date = base;
|
|
68
|
+
value = formatDay(date, slashStyle);
|
|
69
|
+
}
|
|
70
|
+
else if (phrase === 'yesterday') {
|
|
71
|
+
date = new Date(base.getTime() - DAY_MS);
|
|
72
|
+
value = formatDay(date, slashStyle);
|
|
73
|
+
}
|
|
74
|
+
else if (phrase === 'last week') {
|
|
75
|
+
date = new Date(base.getTime() - 7 * DAY_MS);
|
|
76
|
+
value = formatDay(date, slashStyle);
|
|
77
|
+
}
|
|
78
|
+
else if (phrase === 'last month') {
|
|
79
|
+
date = addMonths(base, -1);
|
|
80
|
+
value = formatDay(date, slashStyle).slice(0, 7);
|
|
81
|
+
grain = 'month';
|
|
82
|
+
}
|
|
83
|
+
else if (phrase === 'last year') {
|
|
84
|
+
date = new Date(Date.UTC(base.getUTCFullYear() - 1, 0, 1));
|
|
85
|
+
value = String(base.getUTCFullYear() - 1);
|
|
86
|
+
grain = 'year';
|
|
87
|
+
}
|
|
88
|
+
else {
|
|
89
|
+
const target = WEEKDAYS.indexOf(phrase.slice(5));
|
|
90
|
+
const back = ((base.getUTCDay() - target + 7) % 7) || 7;
|
|
91
|
+
date = new Date(base.getTime() - back * DAY_MS);
|
|
92
|
+
value = formatDay(date, slashStyle);
|
|
93
|
+
}
|
|
94
|
+
found.push({ at: m.index ?? 0, phrase: m[1].replace(/\s+/g, ' '), date, value, grain });
|
|
95
|
+
}
|
|
96
|
+
const seen = new Set();
|
|
97
|
+
return found.sort((a, b) => a.at - b.at).flatMap(f => {
|
|
98
|
+
const key = f.phrase.toLowerCase();
|
|
99
|
+
if (seen.has(key))
|
|
100
|
+
return [];
|
|
101
|
+
seen.add(key);
|
|
102
|
+
return [f];
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
/** The hints for one excerpt: `"three weeks ago" is 2023/05/07 (Sun)`, in the order the phrases appear, each phrase once. */
|
|
106
|
+
function relativeTimeHints(text, sessionDate) {
|
|
107
|
+
return relativeTimePoints(text, sessionDate).map(point => `"${point.phrase}" is ${point.value}`);
|
|
108
|
+
}
|
|
109
|
+
/** The excerpt with its hints on one line below it; the text itself is untouched. An excerpt with no hint is returned as it was. */
|
|
110
|
+
function withRelativeTimeHints(text, sessionDate) {
|
|
111
|
+
const hints = relativeTimeHints(text, sessionDate);
|
|
112
|
+
return hints.length === 0 ? text : `${text}\n\n[Counted from this session's date, ${sessionDate}: ${hints.join('; ')}]`;
|
|
113
|
+
}
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.byPlace = byPlace;
|
|
4
|
+
exports.byScoreThenPlace = byScoreThenPlace;
|
|
5
|
+
exports.semanticSearch = semanticSearch;
|
|
6
|
+
const readCache_1 = require("../db/readCache");
|
|
7
|
+
const statements_1 = require("../db/statements");
|
|
8
|
+
/**
|
|
9
|
+
* Note path, then chunk id: the order of chunks whose scores are equal. A note's chunks are written top to bottom (by
|
|
10
|
+
* the indexer, and by the importer, which copies them in that order), so within one note the id is the chunk's place,
|
|
11
|
+
* the same in every index of the same notes. Across notes ids are not: they follow the order indexing wrote the notes
|
|
12
|
+
* in, which concurrent indexing does not fix.
|
|
13
|
+
*/
|
|
14
|
+
function byPlace(a, b) {
|
|
15
|
+
if (a.notePath !== b.notePath)
|
|
16
|
+
return a.notePath < b.notePath ? -1 : 1;
|
|
17
|
+
return a.chunkId - b.chunkId;
|
|
18
|
+
}
|
|
19
|
+
/** Highest score first; equal scores by place (`byPlace`). */
|
|
20
|
+
function byScoreThenPlace(a, b) {
|
|
21
|
+
return b.score - a.score || byPlace(a, b);
|
|
22
|
+
}
|
|
23
|
+
async function semanticSearch(db, vectorIndex, provider, query, topK, probe,
|
|
24
|
+
/** Only the chunks it shows are candidates: the nearest `topK` of them, however many others are nearer. */
|
|
25
|
+
scope) {
|
|
26
|
+
// A fresh/empty memory has nothing to compare against. Avoid paying the
|
|
27
|
+
// provider/network cost for a query embedding that cannot produce a hit.
|
|
28
|
+
// Ask vec0's rowid shadow table, not the virtual table: `SELECT ... FROM chunk_embeddings` without a
|
|
29
|
+
// MATCH walks every vector (milliseconds at 10,000 notes, growing with the index) to learn "not empty".
|
|
30
|
+
// `chunk_embeddings_rowids` holds one row per stored vector.
|
|
31
|
+
const hasVectors = (0, statements_1.prepareOnce)(db, 'SELECT 1 FROM chunk_embeddings_rowids LIMIT 1').get();
|
|
32
|
+
if (!hasVectors || topK <= 0)
|
|
33
|
+
return [];
|
|
34
|
+
const queryEmbedding = provider.embedQuery
|
|
35
|
+
? await provider.embedQuery(query)
|
|
36
|
+
: (await provider.embed([query]))[0];
|
|
37
|
+
if (probe)
|
|
38
|
+
probe.queryVector = queryEmbedding;
|
|
39
|
+
// The chunks of the `k` nearest vectors (of those `among` names, when it is given), best first, and how many vectors
|
|
40
|
+
// were read: more than `k` when the farthest of them is tied, and the cut is made by place, not by where vectors are
|
|
41
|
+
// stored; fewer only when there are no more.
|
|
42
|
+
const nearest = (k, among) => {
|
|
43
|
+
const hits = vectorIndex.search(queryEmbedding, k, among);
|
|
44
|
+
if (hits.length === 0)
|
|
45
|
+
return { vectors: 0, chunks: [] };
|
|
46
|
+
const scoreMap = new Map(hits.map(h => [h.chunkId, h.score]));
|
|
47
|
+
const rows = (0, statements_1.prepareOnce)(db, `SELECT id, note_path, heading_path, text, start_line, end_line FROM chunks WHERE id IN (SELECT value FROM json_each(?))`).all(JSON.stringify(hits.map(h => h.chunkId)));
|
|
48
|
+
return {
|
|
49
|
+
vectors: hits.length,
|
|
50
|
+
chunks: rows
|
|
51
|
+
.map(r => ({ notePath: r.note_path, chunkId: r.id, headingPath: r.heading_path, text: r.text, score: scoreMap.get(r.id) ?? 0, startLine: r.start_line, endLine: r.end_line }))
|
|
52
|
+
.sort(byScoreThenPlace),
|
|
53
|
+
};
|
|
54
|
+
};
|
|
55
|
+
// The query is embedded: nothing below waits for anything, so the nearest chunks are read in one transaction (inReadSnapshot).
|
|
56
|
+
return (0, readCache_1.inReadSnapshot)(db, () => {
|
|
57
|
+
if (!scope)
|
|
58
|
+
return nearest(topK).chunks.slice(0, topK);
|
|
59
|
+
// Judged before the cut: when fewer than `topK` of the nearest chunks are shown (other chunks are nearer), the nearest
|
|
60
|
+
// of the shown chunks themselves are read instead, so however many others are nearer, none takes their places.
|
|
61
|
+
const first = nearest(topK);
|
|
62
|
+
const shown = first.chunks.filter(scope.shows);
|
|
63
|
+
if (shown.length >= topK || first.vectors < topK)
|
|
64
|
+
return shown.slice(0, topK);
|
|
65
|
+
return nearest(topK, scope.chunkIds()).chunks.slice(0, topK);
|
|
66
|
+
});
|
|
67
|
+
}
|