renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,1224 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.ENTITY_LANE_RESERVED = exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS = exports.SESSION_EXPAND_DEFAULT_NOTES = exports.SESSION_EXPAND_REASON = exports.SESSION_EXPAND_RANKED_TOKENS = exports.SESSION_EXPAND_DEFAULT_TOKENS = exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = exports.SESSION_DIVERSITY_DEFAULT_CAP = exports.WRITE_CLOCK_TOLERANCE_MS = void 0;
4
- exports.searchBestFirstEnabled = searchBestFirstEnabled;
5
- exports.parseQueryDateRef = parseQueryDateRef;
6
- exports.queryDateRefMs = queryDateRefMs;
7
- exports.hasNowOverride = hasNowOverride;
8
- exports.asofPastLike = asofPastLike;
9
- exports.asofDatedRef = asofDatedRef;
10
- exports.hasPastIntent = hasPastIntent;
11
- exports.hasStrictPastIntent = hasStrictPastIntent;
12
- exports.frontmatterMarksHistorical = frontmatterMarksHistorical;
13
- exports.sessionDateMs = sessionDateMs;
14
- exports.resolveSessionDate = resolveSessionDate;
15
- exports.clampSessionDate = clampSessionDate;
16
- exports.boundContentDate = boundContentDate;
17
- exports.createNoteTimeBound = createNoteTimeBound;
18
- exports.createNoteDater = createNoteDater;
19
- exports.orderHits = orderHits;
20
- exports.contentfulText = contentfulText;
21
- exports.isSpeakerStub = isSpeakerStub;
22
- exports.resolveSessionDiversityCap = resolveSessionDiversityCap;
23
- exports.resolveSessionExpand = resolveSessionExpand;
24
- exports.resolveRankedExpansion = resolveRankedExpansion;
25
- exports.rankedStatementTokens = rankedStatementTokens;
26
- exports.calibratedModelKey = calibratedModelKey;
27
- exports.retrieveContext = retrieveContext;
28
- const ledgerLiveness_1 = require("./ledgerLiveness");
29
- const ledgerPresentation_1 = require("./ledgerPresentation");
30
- const ledgerEventTime_1 = require("./ledgerEventTime");
31
- const dailyEntryTime_1 = require("./dailyEntryTime");
32
- const dailyNote_1 = require("../memory/dailyNote");
33
- const readCache_1 = require("../db/readCache");
34
- const statements_1 = require("../db/statements");
35
- const semantic_1 = require("./semantic");
36
- const entityLookup_1 = require("./entityLookup");
37
- const factsFirst_1 = require("./factsFirst");
38
- const conversationShape_1 = require("./conversationShape");
39
- const contextRelevance_1 = require("../mvp/contextRelevance");
40
- const keyword_1 = require("./keyword");
41
- const queryHelpers_1 = require("./queryHelpers");
42
- const ledgerPresentation_2 = require("./ledgerPresentation");
43
- const plainWording_1 = require("./plainWording");
44
- const rankingTrace_1 = require("./rankingTrace");
45
- function envFlagOn(name) {
46
- const v = (process.env[name] ?? '').trim().toLowerCase();
47
- return v === '1' || v === 'true' || v === 'yes' || v === 'on';
48
- }
49
- /** Newest-first is product default (S0 KEEP). Opt out with RENOR_NEWEST_FIRST=0/false/off. */
50
- function newestFirstEnabled() {
51
- const v = (process.env.RENOR_NEWEST_FIRST ?? '').trim().toLowerCase();
52
- if (v === '0' || v === 'false' || v === 'no' || v === 'off')
53
- return false;
54
- if (v === '')
55
- return true;
56
- return envFlagOn('RENOR_NEWEST_FIRST');
57
- }
58
- /**
59
- * RENOR_SEARCH_BEST_FIRST (default off; `1` / `true` / `yes` / `on` turns it on): search_memory lays its window out best
60
- * match first (`rankedBy: "score"`) instead of newest session first. It reaches only the reads that ask for it
61
- * (`RetrieveOptions.bestFirst`, which only the search_memory tool sets): get_context, the CLI and RENOR_NEWEST_FIRST are
62
- * untouched, and so is which chunks are selected. Every hit keeps its session_date. Measured on the agent-workflow eval.
63
- */
64
- function searchBestFirstEnabled(env = process.env) {
65
- const v = (env.RENOR_SEARCH_BEST_FIRST ?? '').trim().toLowerCase();
66
- return v === '1' || v === 'true' || v === 'yes' || v === 'on';
67
- }
68
- /**
69
- * Temporal ranking is product default (RenorBench prefer_current/prefer_historical
70
- * fix, 2026-09-17). Opt out with RENOR_TEMPORAL_RANKING=0/false/off — a kill
71
- * switch that reverts to pre-temporal ordering without a redeploy.
72
- */
73
- function temporalRankingEnabled() {
74
- const v = (process.env.RENOR_TEMPORAL_RANKING ?? '').trim().toLowerCase();
75
- if (v === '0' || v === 'false' || v === 'no' || v === 'off')
76
- return false;
77
- if (v === '')
78
- return true;
79
- return envFlagOn('RENOR_TEMPORAL_RANKING');
80
- }
81
- /**
82
- * Explicit-date as-of lane (RENOR_TEMPORAL_ASOF, default OFF).
83
- * Opt in with RENOR_TEMPORAL_ASOF=1/true/yes/on — a kill switch that restores
84
- * pre-lane behavior without a redeploy.
85
- */
86
- function asofEnabled() {
87
- return envFlagOn('RENOR_TEMPORAL_ASOF');
88
- }
89
- const ASOF_MONTHS = {
90
- january: 1, february: 2, march: 3, april: 4, may: 5, june: 6,
91
- july: 7, august: 8, september: 9, october: 10, november: 11, december: 12,
92
- };
93
- /**
94
- * Extract an explicit past-date reference from a natural-language query.
95
- * General (gold-blind): matches any month+year, season+year, or bare year —
96
- * never fixture-specific dates. Returns undefined when the query carries no
97
- * calendar reference or origin marker.
98
- */
99
- function parseQueryDateRef(query) {
100
- const q = query.toLowerCase();
101
- const monthNames = Object.keys(ASOF_MONTHS).join('|');
102
- const my = q.match(new RegExp(`\\b(${monthNames})\\s+(\\d{4})\\b`));
103
- if (my)
104
- return { year: Number(my[2]), month: ASOF_MONTHS[my[1]], originMarker: false };
105
- const season = q.match(/\b(early|mid|late)\s+(\d{4})\b/);
106
- if (season) {
107
- const month = season[1] === 'early' ? 3 : season[1] === 'mid' ? 6 : 9;
108
- return { year: Number(season[2]), month, originMarker: false };
109
- }
110
- const part = q.match(/\b(end|start|beginning|middle)\s+of\s+(\d{4})\b/);
111
- if (part) {
112
- const month = part[1] === 'end' ? 12 : part[1] === 'middle' ? 6 : 1;
113
- return { year: Number(part[2]), month, originMarker: false };
114
- }
115
- const iso = q.match(/\b(\d{4})-(\d{2})-(\d{2})\b/);
116
- if (iso)
117
- return { year: Number(iso[1]), month: Number(iso[2]), originMarker: false };
118
- const bareYear = q.match(/\b((?:19|20)\d{2})\b/);
119
- if (bareYear)
120
- return { year: Number(bareYear[1]), originMarker: false };
121
- if (/\boriginal(ly)?\b|\bv1\b|first version/.test(q))
122
- return { year: 0, originMarker: true };
123
- return undefined;
124
- }
125
- /** Resolve a date ref to an end-of-period timestamp for past/future tests. */
126
- function queryDateRefMs(ref) {
127
- if (ref.originMarker)
128
- return Number.NEGATIVE_INFINITY; // always "the past"
129
- const month = ref.month ?? 12;
130
- // End of the referenced month (Date.UTC month is 0-based; day 0 = last of prev).
131
- return Date.UTC(ref.year, month, 0, 23, 59, 59);
132
- }
133
- function hasNowOverride(query) {
134
- return /\b(now|currently|current)\b/i.test(query);
135
- }
136
- /**
137
- * Past-like via the as-of lane: an explicit past calendar reference (or an
138
- * origin marker) with no present-tense override. Independent of the temporal
139
- * flag — the lane has its own kill switch.
140
- */
141
- function asofPastLike(query, nowMs) {
142
- if (!asofEnabled())
143
- return false;
144
- const ref = parseQueryDateRef(query);
145
- if (!ref)
146
- return false;
147
- if (hasNowOverride(query))
148
- return false;
149
- return queryDateRefMs(ref) < nowMs - 86_400_000;
150
- }
151
- /** Explicit past-date refs (no origin markers) for the date-overlap boost. */
152
- function asofDatedRef(query, nowMs) {
153
- if (!asofEnabled())
154
- return undefined;
155
- const ref = parseQueryDateRef(query);
156
- if (!ref || ref.originMarker)
157
- return undefined;
158
- if (hasNowOverride(query))
159
- return undefined;
160
- if (!(queryDateRefMs(ref) < nowMs - 86_400_000))
161
- return undefined;
162
- return ref;
163
- }
164
- /** Explicit past-intent query: "what was it before / previously / used to be". */
165
- function hasPastIntent(query) {
166
- return /\b(previously|before|used to|former|formerly|earlier|in the past|history)\b/i.test(query);
167
- }
168
- /**
169
- * STRICT past intent: the temporal word must signal that the QUESTION TARGET is
170
- * the past state, not merely appear in a query whose answer is current
171
- * ("Which FORMER vegetarian NOW eats fish?" asks for the present).
172
- */
173
- function hasStrictPastIntent(query) {
174
- if (!hasPastIntent(query))
175
- return false;
176
- return !/\b(now|currently|current)\b/i.test(query);
177
- }
178
- /**
179
- * Corpus-level historical marker in note frontmatter. Deliberately conservative:
180
- * only explicit declarations count, so ordinary notes are never demoted.
181
- */
182
- function frontmatterMarksHistorical(frontmatterJson) {
183
- if (!frontmatterJson)
184
- return false;
185
- let fm = {};
186
- try {
187
- const parsed = JSON.parse(frontmatterJson);
188
- if (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
189
- fm = parsed;
190
- }
191
- catch {
192
- return false;
193
- }
194
- const historicalFlag = fm['historical'] === true || fm['archived'] === true;
195
- const kind = typeof fm['kind'] === 'string' ? fm['kind'].toLowerCase() : '';
196
- const type = typeof fm['type'] === 'string' ? fm['type'].toLowerCase() : '';
197
- return historicalFlag || kind === 'history' || kind === 'historical' || type === 'history' || type === 'historical';
198
- }
199
- function asDateString(value) {
200
- if (typeof value === 'string' && value.trim())
201
- return value.trim();
202
- if (value instanceof Date && !Number.isNaN(value.getTime()))
203
- return value.toISOString();
204
- if (typeof value === 'number' && Number.isFinite(value)) {
205
- const d = new Date(value);
206
- if (!Number.isNaN(d.getTime()))
207
- return d.toISOString();
208
- }
209
- return undefined;
210
- }
211
- /**
212
- * Parse LME `YYYY/MM/DD (Dow) HH:MM` as UTC and ISO date-only strings as UTC midnight.
213
- * ISO datetimes require `Z` or an explicit numeric offset; never interpret them in host-local time.
214
- */
215
- function sessionDateMs(value) {
216
- if (!value)
217
- return Number.NEGATIVE_INFINITY;
218
- const official = /^(\d{4})\/(\d{2})\/(\d{2})(?: \([A-Za-z]{3}\))?(?: (\d{2}):(\d{2}))?$/.exec(value);
219
- const isoDateOnly = /^\d{4}-\d{2}-\d{2}$/.test(value);
220
- const zonedIsoDateTime = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?(?:Z|[+-]\d{2}:\d{2})$/i.test(value);
221
- const iso = official
222
- ? `${official[1]}-${official[2]}-${official[3]}T${official[4] ?? '00'}:${official[5] ?? '00'}:00Z`
223
- : isoDateOnly
224
- ? `${value}T00:00:00Z`
225
- : zonedIsoDateTime
226
- ? value
227
- : undefined;
228
- const ms = iso === undefined ? Number.NaN : Date.parse(iso);
229
- return Number.isFinite(ms) ? ms : Number.NEGATIVE_INFINITY;
230
- }
231
- function resolveSessionDate(frontmatterJson, modifiedAt) {
232
- let fm = {};
233
- if (frontmatterJson) {
234
- try {
235
- const parsed = JSON.parse(frontmatterJson);
236
- if (parsed && typeof parsed === 'object' && !Array.isArray(parsed))
237
- fm = parsed;
238
- }
239
- catch {
240
- fm = {};
241
- }
242
- }
243
- const modified = asDateString(modifiedAt);
244
- const asserted = asDateString(fm.session_date) ?? asDateString(fm.date);
245
- if (asserted === undefined)
246
- return modified;
247
- return clampSessionDate(asserted, modified);
248
- }
249
- // Absorbs coarse kernel file timestamps and 1–2 s filesystem resolution.
250
- exports.WRITE_CLOCK_TOLERANCE_MS = 2000;
251
- /** Every content-derived clock, including entry/event overrides, obeys the write-time bound plus clock tolerance. */
252
- function clampSessionDate(asserted, modifiedAt) {
253
- const modified = asDateString(modifiedAt);
254
- const modifiedMs = sessionDateMs(modified);
255
- // With no trusted timestamp, untrusted content cannot gain newest-session priority.
256
- if (!Number.isFinite(modifiedMs))
257
- return undefined;
258
- if (asserted === undefined)
259
- return modified;
260
- const assertedMs = sessionDateMs(asserted);
261
- // A claim that is not an instant sorts as undated (sessionDateMs is -Infinity), so it is shown as undated, never verbatim.
262
- if (!Number.isFinite(assertedMs))
263
- return undefined;
264
- return assertedMs > modifiedMs + exports.WRITE_CLOCK_TOLERANCE_MS ? modified : asserted;
265
- }
266
- /**
267
- * Bound a content-derived date that is stored (structured-row valid_from) by the source file's write time, with the same
268
- * tolerance as `clampSessionDate`. Day-precision dates stay day-precision, and a clamped one floors to the write day, so
269
- * mixed formats still compare consistently. With no trusted write time, or a date that cannot be read as an instant, no
270
- * content date is kept (unknown stays unknown).
271
- */
272
- function boundContentDate(asserted, modifiedAt) {
273
- if (asserted === null)
274
- return null;
275
- const modified = asDateString(modifiedAt ?? undefined);
276
- if (!Number.isFinite(sessionDateMs(modified)))
277
- return null;
278
- const trimmed = asserted.trim();
279
- // A zone-less date-time reads as UTC, the way every other date in the index does; sessionDateMs refuses it otherwise.
280
- const comparable = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?$/.test(trimmed) ? `${trimmed}Z` : trimmed;
281
- if (!Number.isFinite(sessionDateMs(comparable)))
282
- return null;
283
- if (clampSessionDate(comparable, modified) === comparable)
284
- return trimmed;
285
- return /^\d{4}-\d{2}-\d{2}$/.test(trimmed) ? modified.slice(0, 10) : modified;
286
- }
287
- /** Bound an excerpt clock using its indexed file timestamp, rather than its (possibly historical) front matter. */
288
- function createNoteTimeBound(db) {
289
- const row = onceStatement(db, 'SELECT modified_at FROM notes WHERE path = ?');
290
- return (notePath, asserted) => {
291
- const note = row().get(notePath);
292
- return clampSessionDate(asserted, note?.modified_at);
293
- };
294
- }
295
- /**
296
- * A note's date as retrieval dates the excerpts of it (front-matter session_date, else date, capped at the file's
297
- * modification time; absent front matter falls back to that time). For a caller that works on whole notes, such as
298
- * evidence hydration, and so needs the date of the note rather than of one excerpt: for a ledger event those differ.
299
- * One prepared statement per dater.
300
- */
301
- function createNoteDater(db) {
302
- const row = onceStatement(db, 'SELECT modified_at, frontmatter_json FROM notes WHERE path = ?');
303
- return notePath => {
304
- const note = row().get(notePath);
305
- return resolveSessionDate(note?.frontmatter_json, note?.modified_at);
306
- };
307
- }
308
- /**
309
- * Lay hits out by the key they are said to be in order of, so the claim is true of the list and not only of
310
- * the code that usually produces it. Stable, and the input is not touched.
311
- * `session_date`: newest first; hits on one date, and hits with no date (which come last), by score.
312
- * `score`: highest first.
313
- */
314
- function orderHits(hits, by) {
315
- if (by === 'score')
316
- return [...hits].sort((a, b) => b.score - a.score);
317
- // A date is read once for each hit, not once for each comparison the sort makes (a few hundred for a window).
318
- return hits.map(hit => ({ hit, ms: sessionDateMs(hit.session_date) })).sort((a, b) => {
319
- const delta = b.ms - a.ms;
320
- return delta !== 0 && !Number.isNaN(delta) ? delta : b.hit.score - a.hit.score; // NaN: both undated
321
- }).map(entry => entry.hit);
322
- }
323
- /**
324
- * The visible text of each hit text the current read has met. A read asks it of the same text several times: whether the
325
- * hit is a stub, the key that drops a repeat, the boost for a hit with something to say, and again for a hit the session
326
- * expansion brings back. The text is the whole input, so the answer is kept until the read ends (`retrieveContext`
327
- * forgets it as it ends and again as the next one starts, which also covers a read that failed) and no longer: nothing
328
- * outlives the read it was worked out for. A text over the size limit, or a read that meets more texts than the count
329
- * limit, is simply worked out again.
330
- */
331
- const CONTENTFUL_HELD_MAX = 256;
332
- const CONTENTFUL_HELD_MAX_CHARS = 8 * 1024;
333
- const contentfulHeld = new Map();
334
- /** Visible text after markdown/speaker-label strip — used to bury transcript stubs. */
335
- function contentfulText(text) {
336
- const raw = text ?? '';
337
- const held = contentfulHeld.get(raw);
338
- if (held !== undefined)
339
- return held;
340
- const visible = stripToContentful(raw);
341
- if (raw.length <= CONTENTFUL_HELD_MAX_CHARS) {
342
- if (contentfulHeld.size >= CONTENTFUL_HELD_MAX)
343
- contentfulHeld.delete(contentfulHeld.keys().next().value);
344
- contentfulHeld.set(raw, visible);
345
- }
346
- return visible;
347
- }
348
- function stripToContentful(raw) {
349
- // Each pass below but the heading rule can only change a text that holds the character it looks for, and most hit texts hold
350
- // few of them, so a pass whose character is absent is not run (a replace that finds nothing still reads the whole text).
351
- let t = raw.includes('\r') ? raw.replace(/\r/g, '') : raw;
352
- if (t.includes('*'))
353
- t = t.replace(/\*+/g, '');
354
- // Whole ATX heading line is structure, not answer text (chunker prefixes
355
- // section buffers with the heading — Kimi R-O1).
356
- t = t.replace(/^#{1,6}\s+.*$/gm, '');
357
- // Speaker-only lines: **Rowan:** / Rowan: / **Gabriel:** / 李雷: — any script and
358
- // either colon. Cue detection was Latin-1 only (`[A-Za-zÀ-ÿ]`), so labels in
359
- // non-Latin scripts were counted as content by contentfulText, which feeds both
360
- // contentBoost and dedupeHits.
361
- if (t.includes(':') || t.includes(':'))
362
- t = t.replace(/^[\s>*-]*[\p{L}\p{M}][\p{L}\p{M} .'-]{0,40}[::]\s*$/gmu, '');
363
- if (t.includes('\n\n'))
364
- t = t.replace(/\n{2,}/g, '\n');
365
- return t.trim();
366
- }
367
- function isSpeakerStub(text) {
368
- const raw = (text ?? '').replace(/\r/g, '').trim();
369
- // Whole chunk is only a speaker cue (optional markdown/list markers) — any
370
- // script, either colon.
371
- if (/^(?:[\s>*-]*\*?\*?[\p{L}\p{M}][\p{L}\p{M} .'-]{0,40}\*?\*?[::]\s*)+$/u.test(raw) && contentfulText(raw).length === 0) {
372
- return true;
373
- }
374
- return contentfulText(raw).length === 0;
375
- }
376
- function contentBoost(text) {
377
- const n = contentfulText(text).length;
378
- if (n < 40)
379
- return 0;
380
- return Math.min(0.12, (n / 800) * 0.12);
381
- }
382
- /** Two hits of one note whose visible text opens the same way are one hit: the key dedupeHits keeps the first of. */
383
- function hitKey(h) {
384
- return `${h.notePath}\0${contentfulText(h.text).slice(0, 80)}`;
385
- }
386
- function dedupeHits(hits) {
387
- const seen = new Set();
388
- const out = [];
389
- for (const h of hits) {
390
- const key = hitKey(h);
391
- if (seen.has(key))
392
- continue;
393
- seen.add(key);
394
- out.push(h);
395
- }
396
- return out;
397
- }
398
- /** The search cap when RENOR_SESSION_DIVERSITY is unset or empty; get_context uses 0. */
399
- exports.SESSION_DIVERSITY_DEFAULT_CAP = 2;
400
- /**
401
- * RENOR_SESSION_DIVERSITY (search default 2, get_context default 0; explicit 0 turns it off): max chunks per note in the top-K
402
- * window. Capped chunks do not vanish — every hit in the candidate pool is
403
- * scanned, so chunks of notes that have not reached the cap still take their
404
- * slots in rank order, and distinct notes that outrank the surplus are kept.
405
- *
406
- * MEASURED LIMIT, stated because the previous revision of this comment claimed
407
- * the opposite ("so the window keeps topK entries"): the window is NOT
408
- * guaranteed to be full. When the candidate pool is concentrated in fewer than
409
- * topK/cap distinct notes there is nothing else to promote — every surplus hit
410
- * belongs to a note already at its cap by construction — so the window shrinks
411
- * below topK. That is exactly the chunk-heavy corpus this flag targets:
412
- * measured on a 4-notes x 5-chunks pool, topK=10, cap=1 -> 4 slots, cap=2 -> 8
413
- * slots, and on the 90-case screen the delivered context fell 3,791 -> 2,221
414
- * tokens (-41.4%). A true backfill would have to retrieve BEYOND the current
415
- * candidate pool, which this function cannot do and does not claim to.
416
- *
417
- * THE CAP IS AUTHORITATIVE WHENEVER IT IS ARMED, including when the candidate pool is no
418
- * deeper than the window. This function used to early-return on `hits.length <= topK`,
419
- * which skipped the cap entirely for a small or single-note vault (measured: cap=1 over ten
420
- * same-note chunks returned all ten, `kept=10`, while the headline above promises a maximum
421
- * per note). The window then shrinks rather than silently ignoring the setting, which is the
422
- * same trade the concentrated-pool case above already makes: the cap is not a backfill
423
- * guarantee, but it is not optional either. Only `cap < 1` (0, or a value that names no cap)
424
- * short-circuits, so the off path is untouched.
425
- *
426
- * Rationale (measured on full500-count-contract results.jsonl, 2026-09-28):
427
- * the top-20 window carries a mean 9.4 repeated-note slots (all different
428
- * chunks of the same note — dedupeHits only matches identical prefixes), and
429
- * cap=2 frees 6.9 slots on 453/500 cases.
430
- *
431
- * DEFAULT 2 since 2026-10-07 (the captain's call), from the held-out answer confirmation of 2026-10-06: GLM-5.3-Flash
432
- * reading and gemma4:31b judging through the shipped CLI, the 395 held-out LongMemEval-S questions the base answered.
433
- * No cap got 340 right and a cap of 2 got 350 (16 wins, 6 losses, +2.5 points, interval +0.3 to +4.8, p = 0.0525),
434
- * with 6.3% fewer context tokens and no category worse by more than 2 questions. A cap of 1 looked best on dev and lost
435
- * evidence on held-out, which is why 1 is not the default. The record: docs/evals/results/SESSION-DIVERSITY-DEFAULT-20261007.md.
436
- *
437
- * WHAT THE CAP COUNTS, because the evidence covers one kind of note. The cap is about the chunks of one conversation or
438
- * one document crowding out the others, so it counts only those. Three kinds of hit share a note path without being parts
439
- * of one thing said, and are left alone (not counted, not cut): a fact row (a negative chunkId, no chunk of a note; the
440
- * facts-first rows all carry the pseudo-path `_mcp_`), a ledger event (every fact-mode save is an event of the one ledger
441
- * file) and an entry of a daily log (one file per day, one memory per entry: dailyEntryTime.ts). Counted by path they
442
- * were capped at `cap` each, and a saved statement stopped coming back from search_memory once `cap` others of the
443
- * ledger, or of its day, outranked it. A benchmark vault holds none of the three, so what was measured is unchanged;
444
- * docs/evals/results/SESSION-DIVERSITY-DEFAULT-20261007.md has the tests that showed it.
445
- *
446
- * Unset (or empty) is the default. 0, or a value that names no cap (junk, negative, below 1), is off, and off is the
447
- * window as it was before the default existed, byte for byte. Searches default to 2 (search_memory, the CLI search and
448
- * the older API/MCP search servers). get_context passes noteCapWhenUnset=0, preserving its existing window and budget
449
- * assembly. An explicit environment value still wins on every read, as it did before this default changed.
450
- */
451
- function resolveSessionDiversityCap(env = process.env, unsetCap = exports.SESSION_DIVERSITY_DEFAULT_CAP) {
452
- const rawCap = (env.RENOR_SESSION_DIVERSITY ?? '').trim();
453
- const cap = rawCap === '' ? unsetCap : Number(rawCap);
454
- if (!Number.isFinite(cap) || cap < 1)
455
- return 0; // off: 0, or a value that names no cap
456
- return Math.floor(cap);
457
- }
458
- /**
459
- * Whether a hit counts toward its note's cap: a chunk of a conversation or a document. A fact row (a negative chunkId)
460
- * is no chunk of a note and a ledger event is no session's chunk (the two tests sessionExpansionHits ranks notes by),
461
- * and an entry of a daily log is a memory of its own.
462
- */
463
- function countsTowardNoteCap(h) {
464
- return h.chunkId > 0 && (0, ledgerLiveness_1.ledgerEventIds)(h).length === 0 && (0, dailyNote_1.dailyNoteDayName)(h.notePath) === null;
465
- }
466
- function applySessionDiversity(hits, cap, _topK) {
467
- // `cap < 1` is the flag-OFF short-circuit and must stay: 0 restores the window exactly as it was
468
- // before the default existed. The former `|| hits.length <= topK` here was a bypass, not an optimization — it
469
- // skipped the configured cap entirely whenever the pool was no deeper than the window.
470
- if (cap < 1)
471
- return hits;
472
- const kept = [];
473
- const counts = new Map();
474
- // Single pass over the WHOLE pool, in rank order: a hit is kept while its note
475
- // has fewer than `cap` slots already taken (a hit that is no chunk of a note is kept and takes none).
476
- // This is the entire mechanism —
477
- // the "backfill" loop that used to follow it was dead code (every item it
478
- // could reach came from a note already at cap, and counts never decrease, so
479
- // its `continue` was always taken and it never appended anything). It is
480
- // deleted rather than kept, because a loop that cannot run cannot be a
481
- // guarantee, and the docstring above used to rest on it.
482
- for (const h of hits) {
483
- if (!countsTowardNoteCap(h)) {
484
- kept.push(h);
485
- continue;
486
- }
487
- const n = counts.get(h.notePath) ?? 0;
488
- if (n < cap) {
489
- counts.set(h.notePath, n + 1);
490
- kept.push(h);
491
- }
492
- }
493
- return kept;
494
- }
495
- /**
496
- * RENOR_SESSION_EXPAND (default 5 notes; 0 turns it off): add the short statements of the notes retrieval already found.
497
- *
498
- * What it fixes, measured (PageIndex scout, 2026-10-03, on the frozen product-path top-20 of LongMemEval-S): the
499
- * window holds every gold SESSION for 98.3% of the 470 answerable questions but every evidence TURN for only 79.4%.
500
- * The miss is inside a session it found: the note comes back through several of its chunks, but not through the one
501
- * short sentence ("by the way, I ...") that answers the question, because that sentence is not what the query is
502
- * about. Adding the short turns of the five best sessions to the top-20 delivered every evidence turn for 93.6%, with
503
- * no model call. Measured through the shipped CLI with this code (results/SESSION-EXPAND-DELIVERY-20261003.md): off
504
- * 373/470 = 79.4%; five notes 440/470 = 93.6% unbounded and 439/470 = 93.4% under the default 2,000-token budget, at
505
- * 3,430 mean / 5,162 p95 tokens of client context against 1,978 / 4,033 off.
506
- *
507
- * What it does: rank the notes in the window by their best hit, take the best `RENOR_SESSION_EXPAND` of them, and add
508
- * their statements that the window does not already hold, best note first and each note in document order, while
509
- * they fit in `RENOR_SESSION_EXPAND_TOKENS` (default SESSION_EXPAND_DEFAULT_TOKENS). The expansion is the longest run
510
- * of that list that fits: it stops at the first statement that does not. Two bounded queries over `chunks`, no embedding and no
511
- * model call.
512
- *
513
- * - A statement is a whole section (one chunk under a unique heading path), skipped whole when over its limit,
514
- * never cut. Conversation-shaped notes use `RENOR_SESSION_EXPAND_STATEMENT_TOKENS` (default 120); every other
515
- * note keeps the former 200-token limit (`RENOR_SESSION_EXPAND_DOCUMENT_STATEMENT_TOKENS`). The shape comes
516
- * only from the note's own headings (conversationShape.ts), never benchmark labels or front matter.
517
- * The document screen found that applying 100 to all notes lost 186 delivered answer statements. Keeping
518
- * documents at 200 preserved every document reply and 96.2% of the conversation token saving (11.7% overall).
519
- * See docs/evals/results/SESSION-EXPAND-STATEMENT-LIMIT-20261007.md for the measurement and its limits.
520
- * - Role-agnostic within a conversation: both user and assistant statements qualify, regardless of speaker.
521
- * - A union, never a replacement: the window is returned as it was, and the expansion only adds. Replacing the
522
- * window with the expansion lost the questions whose evidence is what the assistant said (56/56 -> 5/56).
523
- * - An added hit was not matched by the query, and says so: `reason` 'session-expand', `score` 0 and no `similarity`
524
- * (get_context's relevance signal stays the window's). get_context surfaces that as a first-class field on the
525
- * item it lists (`matched: false`, mvp/agentMemory.ts normalizeMemory); the search lane is unchanged. It carries its
526
- * note's date, so the layout (`orderHits`) places it with its session, after the chunks that matched; by score it
527
- * comes after the whole window.
528
- * - It never adds a note: every added hit belongs to a note the window holds, so a read scoped by note path keeps or
529
- * drops it with that note. And it never adds a ledger event: the ledger is no session, its events are judged for
530
- * liveness and scope one by one, and an event added here would skip both (a superseded event would come back as
531
- * live). An event in the window does not rank the ledger either.
532
- *
533
- * RANKED (default, for an embedding model with a calibrated bar; RENOR_SESSION_EXPAND_RANK=0 restores the above): the
534
- * statements of EVERY note in the window compete by their cosine to the query (one `VectorIndex.search` among their
535
- * chunk ids: no embedding, no model call), and the best are added while they fit in `SESSION_EXPAND_RANKED_TOKENS`, each
536
- * note's laid out in document order. The statement limits are the former expansion's (120 for a conversation, 200 for a
537
- * document), and only the cosine ranks: nothing the note's author writes (a heading, a speaker label) adds to it. Why:
538
- * the expansion above spends its budget on every short statement of five notes whatever the query, and the gold
539
- * evidence of a multi-session question sits in a session ranked below five. Measured on the held-out 400 in
540
- * docs/evals/results/RANKED-EXPANSION-20261011.md. Armed only for a calibrated model because it relies on the
541
- * cosine's order being meaningful: the lexical eval stub and an unmeasured model keep the expansion above, byte for byte.
542
- * An OpenAI-compatible endpoint that returns a calibrated model's vectors is armed by the operator's declaration
543
- * (RENOR_EMBEDDING_SERVED_BY, see calibratedModelKey); docs/evals/results/EXPANSION-PATH-ALIGNMENT-20261018.md.
544
- *
545
- * Unset is five notes. 0, a value that names no note, a budget that allows nothing or both statement limits admitting
546
- * nothing is off, and off costs nothing: no statement is prepared and the result is what it was before the expansion
547
- * existed, byte for byte. The budget bounds what is added; nothing in the window is cut to make room. A caller that
548
- * cuts the result by position cuts what the layout puts last: by score, the added hits; newest first, the oldest
549
- * session's hits, matched or added. Measured with the benchmark client's 6,000-token cap: five notes lost no question
550
- * at any budget, seven notes unbounded lost 3 whose evidence the window had delivered. And a caller with a small
551
- * budget of its own spends it on added hits too, so it must not let them take a match's room: get_context (800 tokens
552
- * by default) gives them only what its facts and its matches leave (mvp/agentMemory.ts applyBudget).
553
- */
554
- /**
555
- * The longest statement of a conversation-shaped note the expansion adds.
556
- *
557
- * 120, not 100: the evidence turns of the questions that still miss at 100 sit just over it. Measured on LongMemEval-S
558
- * 2026-10-10 (cache-only retrieval ablation, practice split and held-out 400): the practice split's remaining
559
- * multi-session misses hold statements of 101-117 tokens, and every one of them is under 120. The held-out curve at the
560
- * shipped defaults, 376 answerable questions: a cap of 100 delivers every gold evidence turn on 341 questions in 2,822
561
- * mean client tokens, 120 on 349 in 2,921 (+8 questions, +3.5%), 150 on 351 in 3,047 (+10, +8.0%); the 120-to-150 step is
562
- * 2 questions inside the noise (exact McNemar p = 0.500). The 2026-10-07 record
563
- * (docs/evals/results/SESSION-EXPAND-STATEMENT-LIMIT-20261007.md) adds that the answer-level effect of the limit is flat
564
- * from 120 to 200 while the delivered context keeps growing, so 120 is the knee of the curve. Documents keep 200: their
565
- * statements were measured separately and a flat 100 there lost 186 delivered answer statements.
566
- */
567
- exports.SESSION_EXPAND_MAX_CHUNK_TOKENS = 120;
568
- /** The longest statement of a note that is not shaped like a conversation (a document): the limit before the cap. */
569
- exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS = 200;
570
- exports.SESSION_EXPAND_DEFAULT_TOKENS = 2000;
571
- /** The ranked expansion's budget. Its statement limits are the former expansion's (120 for a conversation, 200 for a document). */
572
- exports.SESSION_EXPAND_RANKED_TOKENS = 1300;
573
- /** The `reason` of a hit the expansion added: no word of the query matched it, and it has no score of its own. */
574
- exports.SESSION_EXPAND_REASON = 'session-expand';
575
- /** How many of the window's best notes are expanded when RENOR_SESSION_EXPAND is unset. */
576
- exports.SESSION_EXPAND_DEFAULT_NOTES = 5;
577
- function resolveSessionExpand(env = process.env) {
578
- const rawNotes = (env.RENOR_SESSION_EXPAND ?? '').trim();
579
- const notes = rawNotes === '' ? exports.SESSION_EXPAND_DEFAULT_NOTES : Number(rawNotes);
580
- if (!Number.isFinite(notes) || notes < 1)
581
- return null; // off: 0, or a value that names no note
582
- const rawBudget = (env.RENOR_SESSION_EXPAND_TOKENS ?? '').trim();
583
- const budget = rawBudget === '' ? exports.SESSION_EXPAND_DEFAULT_TOKENS : Number(rawBudget);
584
- if (!Number.isFinite(budget) || budget < 1)
585
- return null; // a budget that allows nothing adds nothing
586
- // Each class has its own limit; one that admits no statement adds nothing for that class, both of them add nothing at all.
587
- const limit = (raw, fallback) => {
588
- const text = (raw ?? '').trim();
589
- const value = text === '' ? fallback : Number(text);
590
- return Number.isFinite(value) && value >= 1 ? Math.floor(value) : 0;
591
- };
592
- const statementTokens = limit(env.RENOR_SESSION_EXPAND_STATEMENT_TOKENS, exports.SESSION_EXPAND_MAX_CHUNK_TOKENS);
593
- const documentStatementTokens = limit(env.RENOR_SESSION_EXPAND_DOCUMENT_STATEMENT_TOKENS, exports.SESSION_EXPAND_DOCUMENT_MAX_CHUNK_TOKENS);
594
- if (statementTokens === 0 && documentStatementTokens === 0)
595
- return null;
596
- return { notes: Math.floor(notes), tokenBudget: Math.floor(budget), statementTokens, documentStatementTokens };
597
- }
598
- function resolveRankedExpansion(env = process.env) {
599
- if ((env.RENOR_SESSION_EXPAND_RANK ?? '').trim() === '0')
600
- return null;
601
- const legacy = resolveSessionExpand(env);
602
- if (!legacy)
603
- return null;
604
- const rawNotes = (env.RENOR_SESSION_EXPAND ?? '').trim();
605
- const rawBudget = (env.RENOR_SESSION_EXPAND_TOKENS ?? '').trim();
606
- return {
607
- notes: rawNotes === '' ? Number.POSITIVE_INFINITY : legacy.notes,
608
- tokenBudget: rawBudget === '' ? exports.SESSION_EXPAND_RANKED_TOKENS : legacy.tokenBudget,
609
- statementTokens: rankedStatementTokens(env) ?? legacy.statementTokens,
610
- documentStatementTokens: legacy.documentStatementTokens,
611
- };
612
- }
613
- /**
614
- * The longest conversation statement the RANKED expansion adds when nothing says otherwise: 400 tokens. The former expansion
615
- * (a command line that does not declare a calibrated model) keeps RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120).
616
- */
617
- exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS = 400;
618
- /**
619
- * RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS (default 400 since the flip of 2026-10-25): the longest conversation statement,
620
- * in tokens, that the RANKED expansion alone may add. Unset or empty: 400, unless RENOR_SESSION_EXPAND_STATEMENT_TOKENS is set,
621
- * which an operator's explicit value keeps winning as it always did. A number of 1 or more: that limit, over the general one.
622
- * The off switch: `0`, a negative number or anything that is not a number returns null, and the ranked expansion then takes
623
- * RENOR_SESSION_EXPAND_STATEMENT_TOKENS (120 unset), as it did before the flip. The former expansion and the documents are
624
- * never touched by it.
625
- *
626
- * Why 400: the session expansion skips a conversation statement over its limit, whole, and the long user turn that answers a
627
- * question is often just over 120 tokens (501 to 700 characters: missing from the page 5 times in 21, against 6 in 777 for
628
- * shorter ones, on the Haiku default run of 2026-10-23). The chunker leaves no section over 400 tokens as one chunk, so 400 is
629
- * the largest value that changes anything. On the official LongMemEval-S instrument with a Haiku 5.5 reader the ranked
630
- * expansion with 400 scored 433 of 500 and with 120 (the control) 431, against 418, 417 and 423 for the command-line default;
631
- * the four pages only the limit completes are right 4 of 4 with it and 0 of 4 without, and the page is 5.5% smaller:
632
- * engine/docs/evals/results/ranked-statement-limit-20261024/ and LEDGER sections BE and BF.
633
- */
634
- function rankedStatementTokens(env = process.env) {
635
- const raw = (env.RENOR_SESSION_EXPAND_RANKED_STATEMENT_TOKENS ?? '').trim();
636
- if (raw === '')
637
- return (env.RENOR_SESSION_EXPAND_STATEMENT_TOKENS ?? '').trim() === '' ? exports.SESSION_EXPAND_RANKED_STATEMENT_TOKENS : null;
638
- const value = Number(raw);
639
- return Number.isFinite(value) && value >= 1 ? Math.floor(value) : null;
640
- }
641
- /**
642
- * The calibrated key (CALIBRATED_BARS) of the embedding model behind a provider, or null when there is none.
643
- *
644
- * A key names the provider that made the vectors. An OpenAI-compatible endpoint that returns Vertex's gemini-embedding-001
645
- * vectors (the benchmark's embedding cache proxy replays them) declares itself `openai:gemini-embedding-001:768`, which is
646
- * not calibrated, so it keeps the former expansion where a hosted workspace, which calls Vertex itself, runs the ranked one.
647
- * RENOR_EMBEDDING_SERVED_BY (default unset) is the operator's declaration of whose vectors the endpoint returns: `vertex`
648
- * there counts the provider as the calibrated model, but only when the key that makes (the declared kind with the provider's
649
- * own model and dimensions) is in CALIBRATED_BARS. Any other value, model or size changes nothing, and the declaration is the
650
- * operator's word: nothing here checks the vectors. Read here and nowhere else; a hosted workspace is unaffected.
651
- */
652
- function calibratedModelKey(identity, dimensions, env = process.env) {
653
- if (!identity)
654
- return null;
655
- const own = (0, contextRelevance_1.embeddingModelKey)(identity.kind, identity.model, dimensions);
656
- if (own in contextRelevance_1.CALIBRATED_BARS)
657
- return own;
658
- const servedBy = (env.RENOR_EMBEDDING_SERVED_BY ?? '').trim().toLowerCase();
659
- if (servedBy === '')
660
- return null;
661
- const declared = (0, contextRelevance_1.embeddingModelKey)(servedBy, identity.model, dimensions);
662
- return declared in contextRelevance_1.CALIBRATED_BARS ? declared : null;
663
- }
664
- function sessionExpansionHits(db, window, expansion, ranking) {
665
- // A note is ranked by its best chunk in the window. A fact row (chunkId < 0) is no chunk of a note, and a ledger
666
- // event is no session's chunk: neither ranks a note.
667
- const best = new Map();
668
- for (const hit of window) {
669
- if (hit.chunkId > 0 && (0, ledgerLiveness_1.ledgerEventIds)(hit).length === 0 && hit.score > (best.get(hit.notePath)?.score ?? Number.NEGATIVE_INFINITY)) {
670
- best.set(hit.notePath, hit);
671
- }
672
- }
673
- const ranked = ranking?.ranked;
674
- if (ranked)
675
- expansion = { ...expansion, ...ranked };
676
- const notes = [...best.values()].sort((a, b) => b.score - a.score).slice(0, expansion.notes);
677
- if (notes.length === 0)
678
- return [];
679
- const rank = new Map(notes.map((hit, i) => [hit.notePath, i]));
680
- // A statement is a whole section: a chunk that no other chunk of its note shares a heading path with. The chunker
681
- // cuts a section of over 400 tokens into paragraphs under one heading path, and such a piece is a fragment of a long
682
- // answer, not something said in a few words.
683
- const paths = notes.map(hit => hit.notePath);
684
- // What a note is shaped like decides its limit: every heading path of its chunks, not only the statements'.
685
- const headings = new Map();
686
- for (const row of (0, statements_1.listStatement)(db, paths.length, marks => `SELECT note_path AS notePath, heading_path AS headingPath FROM chunks WHERE note_path IN (${marks})`).all(...(0, statements_1.paddedList)(paths))) {
687
- const list = headings.get(row.notePath) ?? [];
688
- list.push(row.headingPath);
689
- headings.set(row.notePath, list);
690
- }
691
- const limitOf = new Map(paths.map(notePath => [
692
- notePath,
693
- (0, conversationShape_1.isConversationShaped)(headings.get(notePath) ?? []) ? expansion.statementTokens : expansion.documentStatementTokens,
694
- ]));
695
- const rows = db.prepare(`SELECT chunkId, notePath, headingPath, text, tokens FROM (
696
- SELECT id AS chunkId, note_path AS notePath, heading_path AS headingPath, text, token_count AS tokens,
697
- start_line AS startLine, COUNT(*) OVER (PARTITION BY note_path, heading_path) AS parts
698
- FROM chunks WHERE note_path IN (${paths.map(() => '?').join(',')})
699
- ) WHERE parts = 1 AND tokens <= ?
700
- ORDER BY startLine, chunkId`).all(...paths, Math.max(expansion.statementTokens, expansion.documentStatementTokens)).filter(row => row.tokens <= limitOf.get(row.notePath))
701
- .sort((a, b) => rank.get(a.notePath) - rank.get(b.notePath)); // stable: document order inside a note
702
- const inWindow = new Set(window.map(hit => hit.chunkId));
703
- const seen = new Set(window.map(hitKey));
704
- const added = [];
705
- let spent = 0;
706
- let candidates = rows;
707
- if (ranking) {
708
- const eligible = rows.filter(row => !inWindow.has(row.chunkId) && !isSpeakerStub(row.text) && (0, ledgerLiveness_1.ledgerEventIds)(row).length === 0 && !seen.has(hitKey(row)));
709
- const cosine = new Map(ranking.vectorIndex.search(ranking.queryVector, Math.max(1, eligible.length), eligible.map(r => r.chunkId)).map(h => [h.chunkId, h.score]));
710
- const picked = [];
711
- let room = expansion.tokenBudget;
712
- // Cosine and nothing else: a heading, a speaker label or a front-matter field is text the note's author wrote, and
713
- // a ranking signal taken from it would be one more way for a planted note to steer what the expansion delivers.
714
- const cosineOf = (row) => cosine.get(row.chunkId) ?? -1;
715
- for (const row of [...eligible].sort((a, b) => cosineOf(b) - cosineOf(a) || a.chunkId - b.chunkId)) {
716
- if (row.tokens > room)
717
- break;
718
- room -= row.tokens;
719
- picked.push(row);
720
- }
721
- const chosen = new Set(picked.map(r => r.chunkId));
722
- candidates = rows.filter(r => chosen.has(r.chunkId)); // note rank, then document order, as the unranked expansion lays them out
723
- }
724
- for (const row of candidates) {
725
- // A ledger event is never added: liveness (closed events) and scope (per event) are judged on the window only.
726
- if (inWindow.has(row.chunkId) || isSpeakerStub(row.text) || (0, ledgerLiveness_1.ledgerEventIds)(row).length > 0)
727
- continue;
728
- const key = hitKey(row);
729
- if (seen.has(key))
730
- continue;
731
- if (spent + row.tokens > expansion.tokenBudget)
732
- break;
733
- spent += row.tokens;
734
- seen.add(key);
735
- const sessionDate = best.get(row.notePath).session_date;
736
- added.push({
737
- notePath: row.notePath,
738
- chunkId: row.chunkId,
739
- headingPath: row.headingPath,
740
- text: row.text,
741
- score: 0,
742
- reason: exports.SESSION_EXPAND_REASON,
743
- ...(sessionDate ? { session_date: sessionDate } : {}),
744
- });
745
- }
746
- return added;
747
- }
748
- /** Prepares `sql` on first use, and once per connection: for lookups that run once per hit, and on every read. */
749
- function onceStatement(db, sql) {
750
- let statement;
751
- return () => (statement ??= (0, statements_1.prepareOnce)(db, sql));
752
- }
753
- function getEntityIdsForNotes(db, notePaths) {
754
- if (notePaths.length === 0)
755
- return [];
756
- return (0, statements_1.listStatement)(db, notePaths.length, marks => `SELECT id, source_note as notePath FROM entities WHERE source_note IN (${marks})`).all(...(0, statements_1.paddedList)(notePaths)).map(r => ({ id: r.id, notePath: r.notePath }));
757
- }
758
- function getRelatedEntityIds(db, entityIds) {
759
- if (entityIds.length === 0)
760
- return [];
761
- const ids = entityIds.map(e => e.id);
762
- const rows = (0, statements_1.listStatement)(db, ids.length, marks => `SELECT DISTINCT target_entity_id as id FROM relations WHERE source_entity_id IN (${marks})
763
- UNION
764
- SELECT DISTINCT source_entity_id as id FROM relations WHERE target_entity_id IN (${marks})`).all(...(0, statements_1.paddedList)(ids), ...(0, statements_1.paddedList)(ids));
765
- const existing = new Set(ids);
766
- return rows.map(r => r.id).filter(id => !existing.has(id));
767
- }
768
- function getNotesForEntities(db, entityIds) {
769
- if (entityIds.length === 0)
770
- return [];
771
- return (0, statements_1.listStatement)(db, entityIds.length, marks => `SELECT DISTINCT source_note FROM entities WHERE id IN (${marks})`).all(...(0, statements_1.paddedList)(entityIds)).map(r => r.source_note);
772
- }
773
- function getBestChunksForNotes(db, notePaths, limit, keep) {
774
- if (notePaths.length === 0)
775
- return [];
776
- // The longest chunk of each note, the first of equally long ones; the notes in path order. Neither is left to row
777
- // order, which follows the order indexing wrote the notes in.
778
- const chunks = (0, statements_1.listStatement)(db, notePaths.length, marks => `SELECT chunkId, notePath, headingPath, text, startLine, endLine FROM (
779
- SELECT id as chunkId, note_path as notePath, heading_path as headingPath, text, start_line as startLine, end_line as endLine,
780
- ROW_NUMBER() OVER (PARTITION BY note_path ORDER BY token_count DESC, id) as rn
781
- FROM chunks WHERE note_path IN (${marks})
782
- ) WHERE rn = 1
783
- ORDER BY notePath
784
- LIMIT ?`).all(...(0, statements_1.paddedList)(notePaths), keep ? -1 : limit);
785
- // Given `keep`, the chunks it shows are judged before the limit (LIMIT -1 is none).
786
- return keep ? chunks.filter(keep).slice(0, limit) : chunks;
787
- }
788
- async function retrieveContext(db, vectorIndex, provider, query, topK = 20, priorityPaths = [], clock = {}, opts = {}) {
789
- contentfulHeld.clear(); // what another read worked out is not this read's
790
- const factHits = (0, factsFirst_1.factsFirstHits)(db, query, Math.min(3, topK), opts.scope?.shows);
791
- const queryProbe = opts.probe ?? {}; // the query vector, for the expansion's ranking
792
- // RENOR_NEAR_DUP_CAP (default off): the pool is read deeper, so the places near-copies leave are taken (plainWording.ts).
793
- const nearDupCap = (0, plainWording_1.resolveNearDupCap)();
794
- const candidateLimit = nearDupCap > 0 ? Math.min(plainWording_1.NEAR_DUP_POOL.max, topK * plainWording_1.NEAR_DUP_POOL.perTopK) : Math.min(40, topK * 4);
795
- // RENOR_QUERY_REWRITE (default off): what is embedded is the query without its conversational wrapper.
796
- const embeddedQuery = (0, plainWording_1.queryRewriteEnabled)() ? (0, plainWording_1.rewriteConversationalQuery)(query) : query;
797
- const semanticHits = vectorIndex && provider
798
- ? await (0, semantic_1.semanticSearch)(db, vectorIndex, provider, embeddedQuery, candidateLimit, queryProbe, opts.scope)
799
- : (() => {
800
- const match = (0, queryHelpers_1.quoteFtsQuery)(query);
801
- if (!match)
802
- return [];
803
- // No embedding provider: the keyword index stands in for the semantic lane. Its rows are judged (ledger
804
- // header and events the words are not in, then the scope) before the page is cut, as every keyword reader does.
805
- return (0, queryHelpers_1.shownPage)(rows => (0, keyword_1.keywordSearch)(db, match, rows), rows => {
806
- const live = (0, ledgerPresentation_2.keywordLedgerHits)(rows.map(r => ({ ...r, score: 1 / (1 + Math.abs(r.score)) })), opts.isLedgerNote, query);
807
- return opts.scope ? live.filter(hit => opts.scope.shows(hit)) : live;
808
- }, candidateLimit);
809
- })();
810
- // RENOR_ENTITY_LANE (default off): the chunks that carry the rare words the query names join the pool at their own cosine.
811
- const laneTerms = (0, plainWording_1.entityLaneEnabled)() && vectorIndex && queryProbe.queryVector
812
- ? (0, readCache_1.inReadSnapshot)(db, () => entityLanePool(db, vectorIndex, queryProbe.queryVector, query, topK, semanticHits, opts))
813
- : [];
814
- // The window is ranked without waiting for anything, so it is read in one transaction (inReadSnapshot).
815
- const window = (0, readCache_1.inReadSnapshot)(db, () => rankWindow({ db, query, topK, priorityPaths, clock, opts, factHits, semanticHits, laneTerms, nearDupCap }));
816
- return finishContext({ db, vectorIndex, provider, query, queryProbe, window, bestFirst: opts.bestFirst === true });
817
- }
818
- /**
819
- * The entity lane (RENOR_ENTITY_LANE): the lane words of the query (plainWording.ts entityLaneTerms) and, appended to
820
- * `pool`, the chunks that carry them and are not in it yet, each at its own cosine to the query vector (one
821
- * `VectorIndex.search` among their ids: no embedding, no model call). A scoped read judges them as it judges the pool.
822
- * Returns the lane words, which the window lifts every hit that carries them by.
823
- */
824
- function entityLanePool(db, vectorIndex, queryVector, query, topK, pool, opts) {
825
- const terms = (0, plainWording_1.entityLaneTerms)(db, query);
826
- if (terms.length === 0)
827
- return [];
828
- const inPool = new Set(pool.map(h => h.chunkId));
829
- const ids = (0, plainWording_1.entityLaneChunkIds)(db, terms, topK, opts.isLedgerNote).filter(id => !inPool.has(id));
830
- if (ids.length === 0)
831
- return terms;
832
- const cosine = new Map(vectorIndex.search(queryVector, ids.length, ids).map(h => [h.chunkId, h.score]));
833
- const rows = (0, statements_1.prepareOnce)(db, 'SELECT id AS chunkId, note_path AS notePath, heading_path AS headingPath, text FROM chunks WHERE id IN (SELECT value FROM json_each(?))').all(JSON.stringify(ids));
834
- for (const row of rows) {
835
- if (!cosine.has(row.chunkId))
836
- continue; // no vector yet: the semantic lane could not have found it either
837
- if (opts.scope && !opts.scope.shows(row))
838
- continue;
839
- pool.push({ ...row, score: cosine.get(row.chunkId) });
840
- }
841
- return terms;
842
- }
843
- /** How many places of a window the entity lane keeps for the best hits that carry every lane word. */
844
- exports.ENTITY_LANE_RESERVED = 3;
845
- /**
846
- * The first `topK` of `ranked`, but with its best ENTITY_LANE_RESERVED hits that carry every lane word in it: each one
847
- * the cut would leave out takes the place of the last hit that carries none. A lift alone depends on the embedding's
848
- * scale (a cosine of 0.8 against 0.1 on the lexical stub, a few hundredths on a real model); a place does not. Without
849
- * lane words, the first `topK`.
850
- */
851
- function reserveLanePlaces(ranked, topK, laneTerms) {
852
- const window = ranked.slice(0, topK);
853
- if (laneTerms.length === 0)
854
- return window;
855
- const carries = (h) => (0, plainWording_1.laneTermShare)(h.text, laneTerms) === 1;
856
- const missing = ranked.filter(carries).slice(0, exports.ENTITY_LANE_RESERVED).filter(h => !window.includes(h));
857
- for (const hit of missing) {
858
- let at = window.length - 1;
859
- while (at >= 0 && (0, plainWording_1.laneTermShare)(window[at].text, laneTerms) > 0)
860
- at--;
861
- if (at < 0)
862
- break;
863
- window.splice(at, 1);
864
- window.push(hit);
865
- }
866
- return window;
867
- }
868
- /** The window of hits of a read: scored, with the graph of its notes, their facts and dates, live and capped per note. Waits for nothing. */
869
- function rankWindow(read) {
870
- const { db, query, topK, priorityPaths, clock, opts, factHits, semanticHits, laneTerms, nearDupCap } = read;
871
- // Set only while the learning log collects the ranker's terms (rankingTrace.ts); otherwise every use below is skipped.
872
- const trace = (0, rankingTrace_1.currentRankingTrace)();
873
- // Raw cosines, kept only for a caller that asked to see them (get_context's relevance signal).
874
- const cosineByChunk = opts.probe ? new Map(semanticHits.map(h => [h.chunkId, h.score])) : null;
875
- const entityHit = (0, entityLookup_1.lookupEntity)(db, query.trim());
876
- // As-of lane pre-compute: explicit past-date queries must not receive the
877
- // "now" recency boost below — the newest note describes the current world,
878
- // not the asked-about past. Gated by its own kill switch.
879
- const asofPreNowMs = clock.nowMs ?? Date.now();
880
- const asofPrePast = asofPastLike(query, asofPreNowMs);
881
- // Prepared once per call, not once per hit: each of these runs for every semantic hit (dozens per call).
882
- const noteRowStmt = onceStatement(db, 'SELECT modified_at, frontmatter_json FROM notes WHERE path = ?');
883
- const ownFactsStmt = onceStatement(db, 'SELECT predicate, object_text, valid_to FROM facts WHERE subject_entity_id = ?');
884
- const noteRows = new Map();
885
- const noteRow = (notePath) => {
886
- if (!noteRows.has(notePath))
887
- noteRows.set(notePath, noteRowStmt().get(notePath));
888
- return noteRows.get(notePath);
889
- };
890
- const ownFactRows = new Map();
891
- const scored = semanticHits.map(h => {
892
- let score = h.score;
893
- let reason = 'semantic';
894
- const terms = trace ? (0, rankingTrace_1.newRankingTerms)() : null;
895
- if (isSpeakerStub(h.text)) {
896
- score -= 0.5;
897
- if (terms)
898
- terms[(0, rankingTrace_1.rankingTermIndex)('stub')] -= 0.5;
899
- reason = 'semantic+stub';
900
- }
901
- else {
902
- const boost = contentBoost(h.text);
903
- if (boost > 0) {
904
- score += boost;
905
- reason += '+content';
906
- if (terms)
907
- terms[(0, rankingTrace_1.rankingTermIndex)('content')] += boost;
908
- }
909
- }
910
- if (priorityPaths.some(p => h.notePath.includes('/' + p) || h.notePath.startsWith(p))) {
911
- score += 0.1;
912
- reason += '+priority';
913
- if (terms)
914
- terms[(0, rankingTrace_1.rankingTermIndex)('priority')] += 0.1;
915
- }
916
- if (entityHit?.sourceNote === h.notePath) {
917
- score += 0.2;
918
- reason += '+entity';
919
- if (terms)
920
- terms[(0, rankingTrace_1.rankingTermIndex)('entity')] += 0.2;
921
- }
922
- if (laneTerms.length > 0) {
923
- const share = (0, plainWording_1.laneTermShare)(h.text, laneTerms);
924
- if (share > 0) {
925
- const boost = plainWording_1.ENTITY_LANE_BOOST * share;
926
- score += boost;
927
- reason += '+entity-lane';
928
- if (terms)
929
- terms[(0, rankingTrace_1.rankingTermIndex)('entity_lane')] += boost;
930
- }
931
- }
932
- // Time-decay recency: +0.1 for today, linearly decaying to 0 over 30 days.
933
- // Skipped for explicit past-date (as-of) queries: recency answers "now",
934
- // the as-of query asks about "then".
935
- const row = noteRow(h.notePath);
936
- const session_date = resolveSessionDate(row?.frontmatter_json, row?.modified_at);
937
- if (row && !asofPrePast) {
938
- const daysSince = ((clock.nowMs ?? Date.now()) - new Date(row.modified_at).getTime()) / 86400000;
939
- const recencyBoost = Math.max(0, 0.1 * (1 - daysSince / 30));
940
- if (recencyBoost > 0.001) {
941
- score += recencyBoost;
942
- reason += '+recency';
943
- if (terms)
944
- terms[(0, rankingTrace_1.rankingTermIndex)('recency')] += recencyBoost;
945
- }
946
- }
947
- const out = { ...h, score, reason, session_date };
948
- return terms ? (0, rankingTrace_1.attachRankingTerms)(out, terms) : out;
949
- });
950
- // Graph expansion
951
- const hitNotePaths = [...new Set(semanticHits.map(h => h.notePath))];
952
- const hitEntityIds = getEntityIdsForNotes(db, hitNotePaths);
953
- const relatedEntityIds = getRelatedEntityIds(db, hitEntityIds);
954
- const relatedNotePaths = getNotesForEntities(db, relatedEntityIds);
955
- const newPaths = relatedNotePaths.filter(p => !hitNotePaths.includes(p));
956
- if (newPaths.length > 0) {
957
- const relatedChunks = getBestChunksForNotes(db, newPaths, topK, opts.scope?.shows);
958
- for (const chunk of relatedChunks) {
959
- const row = noteRow(chunk.notePath);
960
- scored.push({
961
- ...chunk,
962
- score: 0.3,
963
- reason: 'graph',
964
- session_date: resolveSessionDate(row?.frontmatter_json, row?.modified_at),
965
- });
966
- }
967
- }
968
- // Fact-aware boost + temporal ranking. Two complementary signals, both general:
969
- //
970
- // 1. Lifecycle facts (facts table, incl. MCP-written facts resolved by
971
- // canonical_name across entity rows — not just per-note entities): chunks
972
- // carrying a superseded value are demoted, chunks carrying the current
973
- // value are lifted, and explicit past-intent queries invert both.
974
- // 2. Corpus signal (works without an LLM fact extractor — RenorBench, and
975
- // real vaults with a History/ convention): notes whose frontmatter marks
976
- // them historical (kind/type: history|historical, historical: true,
977
- // archived: true) are demoted for present-tense queries and lifted for
978
- // past-intent ones.
979
- // No hardcoded subject or cue table. Kill switch: RENOR_TEMPORAL_RANKING=0.
980
- const temporal = temporalRankingEnabled();
981
- const pastIntent = temporal && hasStrictPastIntent(query);
982
- // As-of lane: explicit past-date queries behave past-like for presentation
983
- // ranking even without past-intent cue words ("in April 2025" has none).
984
- const asofNowMs = clock.nowMs ?? Date.now();
985
- const asofPast = asofPastLike(query, asofNowMs);
986
- const asofRef = asofDatedRef(query, asofNowMs);
987
- const historicalNotes = new Set();
988
- const factsByNote = new Map();
989
- if (temporal) {
990
- // Frontmatter historical flags for all hit notes (one batched query).
991
- if (hitNotePaths.length > 0) {
992
- const noteRows = (0, statements_1.listStatement)(db, hitNotePaths.length, marks => `SELECT path, frontmatter_json FROM notes WHERE path IN (${marks})`).all(...(0, statements_1.paddedList)(hitNotePaths));
993
- for (const row of noteRows) {
994
- if (frontmatterMarksHistorical(row.frontmatter_json))
995
- historicalNotes.add(row.path);
996
- }
997
- }
998
- // Lifecycle facts: a note's entities share canonical names with MCP-written
999
- // entities (source_note '_mcp_'), so expand by canonical_name — otherwise
1000
- // MCP facts would never match corpus hits.
1001
- if (hitEntityIds.length > 0) {
1002
- const ids = hitEntityIds.map(e => e.id);
1003
- const nameRows = (0, statements_1.listStatement)(db, ids.length, marks => `SELECT DISTINCT canonical_name FROM entities WHERE id IN (${marks})`).all(...(0, statements_1.paddedList)(ids));
1004
- const names = nameRows.map(r => String(r.canonical_name)).filter(Boolean);
1005
- if (names.length > 0) {
1006
- const entityRows = (0, statements_1.listStatement)(db, names.length, marks => `SELECT id, canonical_name, source_note FROM entities WHERE canonical_name COLLATE NOCASE IN (${marks})`).all(...(0, statements_1.paddedList)(names));
1007
- const nameByEntityId = new Map(entityRows.map(r => [r.id, String(r.canonical_name).toLowerCase()]));
1008
- const factRows = (0, statements_1.listStatement)(db, entityRows.length, marks => `SELECT subject_entity_id, predicate, object_text, valid_to FROM facts WHERE subject_entity_id IN (${marks})`).all(...(0, statements_1.paddedList)(entityRows.map(r => r.id)));
1009
- for (const hitEntity of hitEntityIds) {
1010
- const ownName = nameByEntityId.get(hitEntity.id);
1011
- if (ownName === undefined)
1012
- continue;
1013
- const merged = factsByNote.get(hitEntity.notePath) ?? [];
1014
- for (const f of factRows) {
1015
- if (nameByEntityId.get(f.subject_entity_id) === ownName)
1016
- merged.push(f);
1017
- }
1018
- factsByNote.set(hitEntity.notePath, merged);
1019
- }
1020
- }
1021
- }
1022
- }
1023
- // Distinctive query words (not pure temporal/function markers): used to anchor
1024
- // the past-intent lift to content — an unconditional lift rockets OTHER
1025
- // subjects' historical notes above multi-constraint gold (RenorBench hop-009).
1026
- const TEMPORAL_STOPWORDS = new Set(['previously', 'before', 'used', 'earlier', 'former', 'formerly', 'past', 'history', 'was', 'were', 'did', 'does', 'the', 'and', 'what', 'where', 'who', 'when', 'from']);
1027
- const contentfulQueryWords = (query.toLowerCase().match(/[\p{L}\p{M}]{3,}/gu) ?? [])
1028
- .filter(w => !TEMPORAL_STOPWORDS.has(w));
1029
- const hitText = (h) => h.text.toLowerCase();
1030
- const queryWords = query.toLowerCase().split(/\s+/);
1031
- for (const hit of scored) {
1032
- // Legacy fact-aware boost: scoped to the hit note's OWN entity facts exactly
1033
- // as before this change — the canonical-name expansion below exists for
1034
- // temporal decisions only and must not perturb the legacy boost distribution.
1035
- const ownEntityFacts = (() => {
1036
- const own = hitEntityIds.find(e => e.notePath === hit.notePath);
1037
- if (!own)
1038
- return [];
1039
- let rows = ownFactRows.get(own.id);
1040
- if (!rows)
1041
- ownFactRows.set(own.id, rows = ownFactsStmt().all(own.id));
1042
- return rows;
1043
- })();
1044
- const facts = factsByNote.get(hit.notePath) ?? [];
1045
- const factMatch = ownEntityFacts.some(f => queryWords.some(w => (f.object_text ?? '').toLowerCase().includes(w) ||
1046
- (f.predicate ?? '').toLowerCase().includes(w)));
1047
- if (factMatch) {
1048
- hit.score += 0.15;
1049
- if (trace)
1050
- (0, rankingTrace_1.addRankingTerm)(hit, 'fact', 0.15);
1051
- hit.reason += '+fact';
1052
- }
1053
- // As-of lane: the query names a past date, so hits carrying the same
1054
- // calendar reference are likelier to hold the as-of answer than the
1055
- // newest note (which describes the current world, not the asked one).
1056
- if (asofRef) {
1057
- const yearToken = String(asofRef.year);
1058
- const lowerText = hit.text.toLowerCase();
1059
- if (lowerText.includes(yearToken)) {
1060
- hit.score += 0.08;
1061
- if (trace)
1062
- (0, rankingTrace_1.addRankingTerm)(hit, 'asof_year', 0.08);
1063
- hit.reason += '+asof-year';
1064
- if (asofRef.month !== undefined) {
1065
- const monthName = Object.keys(ASOF_MONTHS).find(m => ASOF_MONTHS[m] === asofRef.month);
1066
- if (monthName && lowerText.includes(monthName)) {
1067
- hit.score += 0.04;
1068
- if (trace)
1069
- (0, rankingTrace_1.addRankingTerm)(hit, 'asof_month', 0.04);
1070
- hit.reason += '+asof-month';
1071
- }
1072
- }
1073
- }
1074
- }
1075
- if (temporal) {
1076
- const text = hitText(hit);
1077
- // Historical-note signal is lift-only, on strict past-intent queries:
1078
- // a blanket present-tense demotion pushed gold historical notes out of the
1079
- // window for queries like "which tea drinker grew up in Sacramento?" —
1080
- // present-tense phrasing, past answer. The lifecycle-fact staleness
1081
- // demotion below remains the general present-tense mechanism.
1082
- const isHistoricalNote = historicalNotes.has(hit.notePath);
1083
- if (pastIntent && isHistoricalNote) {
1084
- // Content-anchored lift: the historical note must itself match the query's
1085
- // distinctive words (≥2), so an unrelated person's history note — however
1086
- // semantically adjacent — does not ride the intent signal past the gold.
1087
- const contentOverlap = contentfulQueryWords.filter(w => text.includes(w)).length;
1088
- if (contentOverlap >= 2) {
1089
- // Must clear recency (+0.1 max) + fact boost (+0.15) the current note
1090
- // typically carries, or past-intent queries can never win.
1091
- hit.score += 0.3;
1092
- if (trace)
1093
- (0, rankingTrace_1.addRankingTerm)(hit, 'historical_intent', 0.3);
1094
- hit.reason += '+historical-intent';
1095
- }
1096
- }
1097
- const currentPredicates = new Set(facts.filter(f => f.valid_to === null).map(f => String(f.predicate)));
1098
- // Demote: chunk carries a value the lifecycle has since superseded.
1099
- for (const f of facts) {
1100
- if (f.valid_to === null || !currentPredicates.has(String(f.predicate)))
1101
- continue;
1102
- const stale = (f.object_text ?? '').trim().toLowerCase();
1103
- if (stale.length >= 3 && text.includes(stale)) {
1104
- if (pastIntent) {
1105
- hit.score += 0.1;
1106
- if (trace)
1107
- (0, rankingTrace_1.addRankingTerm)(hit, 'past_fact', 0.1);
1108
- hit.reason += '+historical-intent';
1109
- }
1110
- else {
1111
- hit.score -= 0.15;
1112
- if (trace)
1113
- (0, rankingTrace_1.addRankingTerm)(hit, 'stale_fact', -0.15);
1114
- hit.reason += '+stale-fact';
1115
- }
1116
- break; // one demotion per hit; stale signal already applied
1117
- }
1118
- }
1119
- // Lift: chunk carries the currently-valid value for a superseded predicate.
1120
- if (!pastIntent) {
1121
- for (const f of facts) {
1122
- if (f.valid_to !== null)
1123
- continue;
1124
- const wasSuperseded = facts.some(g => g.valid_to !== null && String(g.predicate) === String(f.predicate));
1125
- if (!wasSuperseded)
1126
- continue; // plain current facts already get the fact boost above
1127
- const cur = (f.object_text ?? '').trim().toLowerCase();
1128
- if (cur.length >= 3 && text.includes(cur)) {
1129
- hit.score += 0.1;
1130
- if (trace)
1131
- (0, rankingTrace_1.addRankingTerm)(hit, 'current_fact', 0.1);
1132
- hit.reason += '+current-fact';
1133
- break;
1134
- }
1135
- }
1136
- }
1137
- }
1138
- }
1139
- const ranked = dedupeHits(scored.sort(semantic_1.byScoreThenPlace));
1140
- if (trace)
1141
- ranked.forEach((hit, index) => trace.pool(hit, index + 1));
1142
- // Fail-open floor. The stub filter must never empty the window: a zero-hit
1143
- // retrieval is a hard abort on the product path, because the benchmark client
1144
- // refuses to score a case with no hits ("Retrieval precondition failed: public
1145
- // CLI search returned zero hits"). That is strictly worse than surfacing a stub.
1146
- // Measured 2026-09-29: an arm carrying a metadata-strip variant of this filter
1147
- // died at case 122/500 on exactly this precondition. Fall back to the unfiltered
1148
- // ranking when filtering would leave nothing.
1149
- const filtered = ranked.filter(h => !isSpeakerStub(h.text));
1150
- const contentful = filtered.length > 0 ? filtered : ranked;
1151
- const datedFactHits = factHits.map(h => {
1152
- const row = noteRow(h.notePath);
1153
- return { ...h, session_date: resolveSessionDate(row?.frontmatter_json, row?.modified_at) };
1154
- });
1155
- const seen = new Set(datedFactHits.map(h => h.text));
1156
- // The ledger's own header is no memory: it comes off with the closed events, before the page is cut.
1157
- const mixed = (0, ledgerPresentation_1.dropLedgerPreamble)((0, ledgerLiveness_1.resolveLedgerHits)([...datedFactHits, ...contentful.filter(h => !seen.has(h.text))], (0, ledgerLiveness_1.closedLedgerEvents)(db), opts.includeHistory === true), opts.isLedgerNote);
1158
- const diversityCap = resolveSessionDiversityCap(process.env, opts.noteCapWhenUnset);
1159
- // RENOR_NEAR_DUP_CAP (default off): near-copies are capped before the per-note cap, on the visible text.
1160
- const windowed = reserveLanePlaces(applySessionDiversity((0, plainWording_1.applyNearDupCap)(mixed, nearDupCap, contentfulText), diversityCap, topK), topK, laneTerms);
1161
- // The window's candidates are recorded before they are copied into the plain hits below, which carry no terms.
1162
- if (trace)
1163
- for (const hit of windowed)
1164
- trace.touch(hit);
1165
- let hits = windowed.map(h => ({
1166
- notePath: h.notePath,
1167
- chunkId: h.chunkId,
1168
- headingPath: h.headingPath,
1169
- text: h.text,
1170
- score: h.score,
1171
- reason: h.reason,
1172
- ...(h.session_date ? { session_date: h.session_date } : {}),
1173
- ...(cosineByChunk?.has(h.chunkId) ? { similarity: cosineByChunk.get(h.chunkId) } : {}),
1174
- // A fact an identity question read names its author, so the reader's view can mark it (mvp/factMarkers.ts).
1175
- ...('author' in h && h.author !== undefined ? { author: h.author } : {}),
1176
- }));
1177
- return { hits, noteRow, temporal, pastIntent, asofPast };
1178
- }
1179
- /** The hits dated, joined by the statements of their sessions and laid out. The first wait is for the date of an entry of a daily note. */
1180
- async function finishContext(read) {
1181
- const { db, vectorIndex, provider, query, queryProbe, window, bestFirst } = read;
1182
- const { noteRow, temporal, pastIntent, asofPast } = window;
1183
- let { hits } = window;
1184
- // A hit that is a ledger event is dated by the event, and one that is an entry of a daily note by the entry: each
1185
- // is one note, so the note's date is one date for every event, or every entry of the day, in it. Decided on the
1186
- // text a hit still has after liveness filtering, and only for the hits that are returned. Everything else keeps its
1187
- // note's date; no score is touched. Entry headers, filenames and replayed event times are content too: none may
1188
- // override the indexed file's write-time bound, even on already indexed notes.
1189
- const eventTime = (0, ledgerEventTime_1.createLedgerEventClock)(db);
1190
- const entryTime = (0, dailyEntryTime_1.createDailyEntryClock)();
1191
- // The dates of events are read as the hits are mapped, before any of them waits: in one transaction (inReadSnapshot).
1192
- hits = await Promise.all((0, readCache_1.inReadSnapshot)(db, () => hits.map(async (h) => {
1193
- const derived = eventTime(h) ?? await entryTime(h);
1194
- if (derived === undefined)
1195
- return h;
1196
- const session_date = clampSessionDate(derived, noteRow(h.notePath)?.modified_at);
1197
- return { ...h, session_date };
1198
- })));
1199
- // RENOR_SESSION_EXPAND (default 5 notes, 0 is off): the short statements of the best notes in the window join it,
1200
- // under a token budget. The window itself is not touched; see sessionExpansionHits.
1201
- const expansion = resolveSessionExpand();
1202
- // Ranking by cosine is armed only for an embedding model the product has calibrated (CALIBRATED_BARS): the order
1203
- // of cosines was measured for that model. Any other provider (the lexical eval stub, an unmeasured model) keeps the
1204
- // former expansion, unless the operator declares that its endpoint serves a calibrated model (calibratedModelKey).
1205
- const rankedExpansion = calibratedModelKey(provider?.identity, provider?.dimensions) !== null ? resolveRankedExpansion() : null;
1206
- if (expansion)
1207
- hits = [...hits, ...(0, readCache_1.inReadSnapshot)(db, () => sessionExpansionHits(db, hits, expansion, rankedExpansion && queryProbe.queryVector && vectorIndex ? { vectorIndex, queryVector: queryProbe.queryVector, ranked: rankedExpansion } : undefined))];
1208
- // Default-on (S0 78.4%): lay the already-selected window out newest-session-first.
1209
- // Does not replace scoring or change which chunks enter top-K. Opt out: RENOR_NEWEST_FIRST=0.
1210
- // An explicit past-intent query overrides this presentation rerank — "what was
1211
- // it before" is precisely the case where newest-first is the wrong order.
1212
- // The as-of lane extends the override to explicit past-date queries, which
1213
- // carry no past-intent cue words but ask about the old world all the same.
1214
- // Either way the result says which key it is in order of (`rankedBy`): the order is a presentation
1215
- // choice the agent cannot see from the hits alone, and hits[0] is not "the answer" unless that key is score.
1216
- // A read that asks for best match first (search_memory under RENOR_SEARCH_BEST_FIRST) is laid out by score.
1217
- const newestFirst = !bestFirst && newestFirstEnabled() && !(temporal && pastIntent) && !asofPast;
1218
- const rankedBy = newestFirst ? 'session_date' : 'score';
1219
- contentfulHeld.clear(); // nothing outlives the read it was worked out for
1220
- // The order the reply presents its hits in, so a candidate's place in it is recorded for the learning log (rankingTrace.ts).
1221
- const presented = orderHits(hits, rankedBy);
1222
- (0, rankingTrace_1.currentRankingTrace)()?.presented(presented);
1223
- return { query, hits: presented, rankedBy };
1224
- }