renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,197 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.scoreRetrieval = scoreRetrieval;
4
- const metrics_1 = require("./metrics");
5
- function emptyAggregate() {
6
- return {
7
- count: 0,
8
- answerable: 0,
9
- abstention: 0,
10
- hit1: 0,
11
- hit5: 0,
12
- hit10: 0,
13
- recall1: 0,
14
- recall5: 0,
15
- recall10: 0,
16
- mrr: 0,
17
- ndcg5: 0,
18
- ndcg10: 0,
19
- abstentionEmpty: 0,
20
- supersessionSensitive: 0,
21
- contradictionFirst: 0,
22
- recallAll5: 0,
23
- recallAll10: 0,
24
- forbiddenSensitive: 0,
25
- forbiddenRetrievals: 0,
26
- latencyMs: [],
27
- contextTokens: [],
28
- };
29
- }
30
- function normalized(value) {
31
- return (value ?? '').trim().toLowerCase();
32
- }
33
- function validateCase(test, seenCaseIds) {
34
- if (seenCaseIds.has(test.id)) {
35
- throw new Error(`Duplicate eval case id: ${test.id}`);
36
- }
37
- seenCaseIds.add(test.id);
38
- const seenGold = new Set();
39
- for (const target of test.relevant) {
40
- const key = `${normalized(target.match)}\0${normalized(target.notePath ?? '')}`;
41
- if (seenGold.has(key)) {
42
- throw new Error(`Duplicate gold target in case ${test.id}: ${target.match}`);
43
- }
44
- seenGold.add(key);
45
- }
46
- if (!test.shouldAbstain && test.relevant.length > 0) {
47
- const hasPositive = test.relevant.some(target => (0, metrics_1.roleOf)(target) === 'positive');
48
- if (!hasPositive) {
49
- throw new Error(`Eval case ${test.id} has gold targets but no positive-role target; binary metrics could never credit it.`);
50
- }
51
- }
52
- for (const target of test.requiredEvidence ?? []) {
53
- if ((0, metrics_1.roleOf)(target) === 'forbidden') {
54
- throw new Error(`Eval case ${test.id} marks required evidence as forbidden.`);
55
- }
56
- }
57
- }
58
- function addCase(agg, test, result) {
59
- agg.count += 1;
60
- if (test.forbiddenEvidence && test.forbiddenEvidence.length > 0) {
61
- agg.forbiddenSensitive += 1;
62
- if ((0, metrics_1.forbiddenRetrieved)(result.hits, test.forbiddenEvidence, 10))
63
- agg.forbiddenRetrievals += 1;
64
- }
65
- const abstain = Boolean(test.shouldAbstain) || test.relevant.length === 0;
66
- if (abstain) {
67
- agg.abstention += 1;
68
- if (result.hits.length === 0)
69
- agg.abstentionEmpty += 1;
70
- }
71
- else {
72
- agg.answerable += 1;
73
- agg.hit1 += (0, metrics_1.hitAtK)(result.hits, test.relevant, 1);
74
- agg.hit5 += (0, metrics_1.hitAtK)(result.hits, test.relevant, 5);
75
- agg.hit10 += (0, metrics_1.hitAtK)(result.hits, test.relevant, 10);
76
- agg.recall1 += (0, metrics_1.recallAtK)(result.hits, test.relevant, 1);
77
- agg.recall5 += (0, metrics_1.recallAtK)(result.hits, test.relevant, 5);
78
- agg.recall10 += (0, metrics_1.recallAtK)(result.hits, test.relevant, 10);
79
- agg.mrr += (0, metrics_1.reciprocalRank)(result.hits, test.relevant);
80
- agg.ndcg5 += (0, metrics_1.ndcgAtK)(result.hits, test.relevant, 5);
81
- agg.ndcg10 += (0, metrics_1.ndcgAtK)(result.hits, test.relevant, 10);
82
- const required = test.requiredEvidence
83
- ?? test.relevant.filter(target => (0, metrics_1.roleOf)(target) === 'positive');
84
- if (required.length > 0) {
85
- agg.recallAll5 += (0, metrics_1.recallAllAtK)(result.hits, required, 5);
86
- agg.recallAll10 += (0, metrics_1.recallAllAtK)(result.hits, required, 10);
87
- }
88
- if ((0, metrics_1.contradictionFirst)(result.hits, test.relevant))
89
- agg.contradictionFirst += 1;
90
- if (test.relevant.some(target => (0, metrics_1.roleOf)(target) === 'positive')
91
- && test.relevant.some(target => (0, metrics_1.roleOf)(target) === 'forbidden')) {
92
- agg.supersessionSensitive += 1;
93
- }
94
- }
95
- if (typeof result.latencyMs === 'number')
96
- agg.latencyMs.push(result.latencyMs);
97
- if (typeof result.contextTokens === 'number')
98
- agg.contextTokens.push(result.contextTokens);
99
- }
100
- function round(value) {
101
- return Math.round(value * 10000) / 10000;
102
- }
103
- function mean(values) {
104
- if (values.length === 0)
105
- return null;
106
- return values.reduce((sum, value) => sum + value, 0) / values.length;
107
- }
108
- function percentile(values, p) {
109
- if (values.length === 0)
110
- return null;
111
- const sorted = [...values].sort((a, b) => a - b);
112
- const index = Math.min(sorted.length - 1, Math.ceil(p * sorted.length) - 1);
113
- return sorted[Math.max(0, index)];
114
- }
115
- function summarize(agg) {
116
- const a = Math.max(1, agg.answerable);
117
- const z = Math.max(1, agg.abstention);
118
- return {
119
- cases: agg.count,
120
- answerableCases: agg.answerable,
121
- abstentionCases: agg.abstention,
122
- hitAt1: round(agg.hit1 / a),
123
- hitAt5: round(agg.hit5 / a),
124
- hitAt10: round(agg.hit10 / a),
125
- recallAt1: round(agg.recall1 / a),
126
- recallAt5: round(agg.recall5 / a),
127
- recallAt10: round(agg.recall10 / a),
128
- recallAllAt5: round(agg.recallAll5 / a),
129
- recallAllAt10: round(agg.recallAll10 / a),
130
- mrr: round(agg.mrr / a),
131
- ndcgAt5: round(agg.ndcg5 / a),
132
- ndcgAt10: round(agg.ndcg10 / a),
133
- abstentionEmptyRate: agg.abstention ? round(agg.abstentionEmpty / z) : null,
134
- supersessionSensitiveCases: agg.supersessionSensitive,
135
- contradictionFirstRate: agg.supersessionSensitive
136
- ? round(agg.contradictionFirst / agg.supersessionSensitive)
137
- : null,
138
- forbiddenSensitiveCases: agg.forbiddenSensitive,
139
- forbiddenRetrievalRate: agg.forbiddenSensitive
140
- ? round(agg.forbiddenRetrievals / agg.forbiddenSensitive)
141
- : null,
142
- meanLatencyMs: mean(agg.latencyMs) === null ? null : round(mean(agg.latencyMs)),
143
- p50LatencyMs: percentile(agg.latencyMs, 0.5) === null ? null : round(percentile(agg.latencyMs, 0.5)),
144
- p95LatencyMs: percentile(agg.latencyMs, 0.95) === null ? null : round(percentile(agg.latencyMs, 0.95)),
145
- meanContextTokens: mean(agg.contextTokens) === null ? null : round(mean(agg.contextTokens)),
146
- };
147
- }
148
- function scoreRetrieval(cases, results) {
149
- if (!Array.isArray(cases) || cases.length === 0)
150
- throw new Error('Empty eval dataset');
151
- const seenCaseIds = new Set();
152
- for (const test of cases)
153
- validateCase(test, seenCaseIds);
154
- const seenResultIds = new Set();
155
- for (const result of results) {
156
- if (seenResultIds.has(result.id)) {
157
- throw new Error(`Duplicate result id: ${result.id}`);
158
- }
159
- seenResultIds.add(result.id);
160
- if (!Array.isArray(result.hits))
161
- throw new Error(`Invalid hits for result: ${result.id}`);
162
- // Measurement validity: a nonfinite or negative timing/context sample must
163
- // fail the run, not silently skew mean/p95 latency or cost (independent review).
164
- for (const [key, value] of [['latencyMs', result.latencyMs], ['contextTokens', result.contextTokens]]) {
165
- if (value !== undefined && (!Number.isFinite(value) || value < 0 || (key === 'contextTokens' && !Number.isInteger(value)))) {
166
- throw new Error(`Invalid ${key} for result: ${result.id}`);
167
- }
168
- }
169
- for (const hit of result.hits) {
170
- if ((hit.text !== undefined && typeof hit.text !== 'string')
171
- || (hit.notePath !== undefined && typeof hit.notePath !== 'string')
172
- || (hit.score !== undefined && !Number.isFinite(hit.score))) {
173
- throw new Error(`Invalid hit for result: ${result.id}`);
174
- }
175
- }
176
- }
177
- const byId = new Map(results.map(result => [result.id, result]));
178
- const overall = emptyAggregate();
179
- const categories = new Map();
180
- for (const test of cases) {
181
- const result = byId.get(test.id);
182
- if (!result)
183
- throw new Error(`Missing result for eval case: ${test.id}`);
184
- addCase(overall, test, result);
185
- const category = categories.get(test.category) ?? emptyAggregate();
186
- addCase(category, test, result);
187
- categories.set(test.category, category);
188
- }
189
- const unknownIds = results.filter(result => !seenCaseIds.has(result.id)).map(result => result.id);
190
- if (unknownIds.length) {
191
- throw new Error(`Results contain unknown case ids: ${unknownIds.join(', ')}`);
192
- }
193
- return {
194
- overall: summarize(overall),
195
- byCategory: Object.fromEntries([...categories.entries()].map(([name, agg]) => [name, summarize(agg)])),
196
- };
197
- }
@@ -1,17 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.countTextTokens = countTextTokens;
4
- const gpt_tokenizer_1 = require("gpt-tokenizer");
5
- /**
6
- * Tokens in `text` (o200k, gpt-tokenizer), with special-token spellings counted as the plain text they are.
7
- *
8
- * gpt-tokenizer's default `encode` treats `<|endoftext|>`, `<|im_start|>` and the other special-token spellings as chat
9
- * protocol and THROWS ("Disallowed special token found"). Benchmark haystacks and user notes contain those strings as
10
- * ordinary text: the 2026-10-21 LongMemEval-S nightly crashed at question ~351/500 on a haystack holding `<|endoftext|>`
11
- * (assembleClientContext, run-longmemeval-product.ts). Counting them as text is what the engine already does
12
- * (src/indexer/o200k.ts countO200k), and for text without such a spelling the count is identical to the default `encode`.
13
- */
14
- const PLAIN_TEXT = { disallowedSpecial: new Set() };
15
- function countTextTokens(text) {
16
- return (0, gpt_tokenizer_1.encode)(text, PLAIN_TEXT).length;
17
- }
@@ -1,264 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.EVIDENCE_HIT_PATH = void 0;
4
- exports.searchEvidenceEnabled = searchEvidenceEnabled;
5
- exports.productEvidenceReport = productEvidenceReport;
6
- exports.collectEvidence = collectEvidence;
7
- exports.evidenceItemsToFactRows = evidenceItemsToFactRows;
8
- exports.renderEvidenceTable = renderEvidenceTable;
9
- exports.evidenceItemsToHit = evidenceItemsToHit;
10
- exports.prependEvidenceHit = prependEvidenceHit;
11
- exports.attachSearchEvidence = attachSearchEvidence;
12
- const gate_1 = require("./gate");
13
- const secondPass_1 = require("./secondPass");
14
- const background_1 = require("./background");
15
- const optin_1 = require("./optin");
16
- /**
17
- * Attach H10-v4 second-pass evidence onto the default product search path.
18
- *
19
- * Kill switch: RENOR_SEARCH_EVIDENCE (default ON since the 2026-09-22 owner
20
- * ship decision — the evidence second pass measured +28 net on LongMemEval-S
21
- * product runs: 389/500 evidence vs 361/500 baseline, p≈1e-8, official clean
22
- * re-validation; the kill switch RENOR_SEARCH_EVIDENCE=0 restores the pre-ship
23
- * behavior byte-for-byte). When off, search returns
24
- * the same hits array as before this change. When on, the default search /
25
- * get_context / search_memory path runs the same extractor as the dedicated
26
- * `evidence` tool (full source-note hydration, 20-note backfill, 6k cap,
27
- * content-addressed cache) and, on a successful extraction, prepends one
28
- * synthetic hit so array-only clients still see the table.
29
- *
30
- * Retrieval ranking and stored chunk text are never rewritten.
31
- */
32
- exports.EVIDENCE_HIT_PATH = '_renor/evidence';
33
- function searchEvidenceEnabled(env = process.env) {
34
- const raw = (env.RENOR_SEARCH_EVIDENCE ?? '').trim().toLowerCase();
35
- if (raw === '0' || raw === 'false' || raw === 'no' || raw === 'off')
36
- return false;
37
- return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on' || raw === '';
38
- }
39
- /**
40
- * One sentence per skip reason, saying whether anything is actually missing. The reader is the
41
- * user's agent, which can change no setting: the variable or key behind a state stays in `reason`
42
- * and the engine's docs, not in the sentence.
43
- */
44
- const EVIDENCE_NOTES = {
45
- 'flag-off': 'Evidence second pass is switched off for this deployment. Nothing is missing: the hits below are the whole result.',
46
- 'mode-off': 'Evidence second pass is switched off for this deployment. Nothing is missing: the hits below are the whole result.',
47
- 'extractor-unavailable': 'This deployment has no evidence model configured, so no dated item table is prepended. Nothing is broken and nothing is missing: the hits below are the whole result.',
48
- 'intent': 'The evidence gate did not select this query, so no dated item table was prepended. Nothing is missing: the hits below are the whole result.',
49
- 'no-sessions': 'The ranked hits carried no dated source session to extract items from. Nothing is missing: the hits below are the whole result.',
50
- 'extractor-parse-error': 'The evidence model answered in a shape the parser rejects, so no dated item table was prepended. Retrieval is unaffected: the hits below are the whole result.',
51
- 'extractor-error': 'The evidence model call failed, so no dated item table was prepended. Retrieval is unaffected: the hits below are the whole result.',
52
- // The two below exist only with bounded-wait evidence on (RENOR_EVIDENCE_ASYNC): the table is missing from this
53
- // response, not from the product. get_context quotes them too, so they name no position in the payload.
54
- 'pending': 'The evidence model did not finish within the time this deployment waits for it, so this response has no dated item table. Retrieval is unaffected. The extraction continues in the background; when it succeeds its table is cached, and the same request over the same notes returns it.',
55
- 'queue-full': 'The evidence model already has as many extractions waiting as this deployment allows, so none was started for this request and this response has no dated item table. Retrieval is unaffected. The same request starts one once the queue drains.',
56
- // Only a workspace that opted in to the Ollama extractor has a daily allowance of model calls.
57
- 'daily-cap': 'This workspace has used its daily allowance of evidence-model calls, so this response has no dated item table. Retrieval is unaffected: the hits below are the whole result. The allowance starts again at 00:00 UTC.',
58
- };
59
- const EVIDENCE_NOTE_DEFAULT = 'The evidence second pass ran and extracted no items. Nothing is missing: the hits below are the whole result.';
60
- const EVIDENCE_NOTE_OK = 'The evidence second pass extracted items; the dated item table is prepended as the first hit above.';
61
- function evidenceState(evidence) {
62
- if (evidence.status === 'ok')
63
- return 'ok';
64
- if (evidence.reason === 'flag-off' || evidence.reason === 'mode-off')
65
- return 'off';
66
- if (evidence.reason === 'extractor-unavailable')
67
- return 'unavailable';
68
- if (evidence.reason === 'extractor-parse-error' || evidence.reason === 'extractor-error')
69
- return 'failed';
70
- if (evidence.reason === 'pending')
71
- return 'pending';
72
- if (evidence.reason === 'queue-full')
73
- return 'busy';
74
- if (evidence.reason === 'daily-cap')
75
- return 'capped';
76
- return 'not-applied';
77
- }
78
- /**
79
- * The same attachment, plus `state` and `note`. Nothing existing is renamed,
80
- * dropped or revalued — only the two fields a reader needs to tell an intended
81
- * configuration from a fault.
82
- */
83
- function productEvidenceReport(evidence) {
84
- return {
85
- ...evidence,
86
- state: evidenceState(evidence),
87
- note: evidence.status === 'ok'
88
- ? EVIDENCE_NOTE_OK
89
- : ((evidence.reason !== undefined ? EVIDENCE_NOTES[evidence.reason] : undefined) ?? EVIDENCE_NOTE_DEFAULT),
90
- };
91
- }
92
- async function collectEvidence(args) {
93
- const decision = (0, gate_1.evidenceGate)(args.options.mode, args.query, args.force === true);
94
- if (!decision.run) {
95
- return { status: 'skipped', reason: decision.reason };
96
- }
97
- const sessions = await (0, secondPass_1.loadEvidenceSessionsFromVault)(args.hits, args.vaultPath, {
98
- transformBody: args.transformBody,
99
- maxSessions: args.options.maxSessions,
100
- charCap: args.options.charCap,
101
- });
102
- if (sessions.length === 0) {
103
- return { status: 'skipped', reason: 'no-sessions' };
104
- }
105
- if (!args.extractor) {
106
- return { status: 'skipped', reason: 'extractor-unavailable' };
107
- }
108
- const extractor = args.extractor;
109
- // The cached table at once, else the failure this very request met within EVIDENCE_FAILURE_TTL_MS: it is said again
110
- // (fromCache) instead of being paid for again. A table wins over a failure remembered for the same key.
111
- const key = extractor.cacheKey(args.query, sessions);
112
- const cached = extractor.replay(key);
113
- if (cached)
114
- return evidenceOk(decision.reason, cached);
115
- const failed = extractor.replayFailure(key);
116
- if (failed)
117
- return { status: 'skipped', reason: 'extractor-parse-error', fromCache: true, model: failed.model, error: failed.error };
118
- const extract = () => extractEvidence(extractor, key, args.query, sessions, decision.reason, args.parseRetries);
119
- const background = args.options.background;
120
- if (!background)
121
- return extract();
122
- // Bounded wait (RENOR_EVIDENCE_ASYNC): this prompt's one extraction, waited for at most background.waitMs. The
123
- // flight is keyed by cache file too, so it always fills the cache this search reads.
124
- const flight = background.flights.join(`${args.options.cachePath ?? ''}\0${key}`, extract);
125
- if (!flight)
126
- return { status: 'skipped', reason: 'queue-full' };
127
- background.hold?.(flight);
128
- const settled = await (0, background_1.waitAtMost)(flight, background.waitMs);
129
- if (!settled)
130
- return { status: 'skipped', reason: 'pending' };
131
- // A joined flight may have been started by a search whose gate gave another reason.
132
- return settled.value.status === 'ok' ? { ...settled.value, reason: decision.reason } : settled.value;
133
- }
134
- function evidenceOk(reason, result) {
135
- return {
136
- status: 'ok',
137
- reason,
138
- fromCache: result.fromCache,
139
- model: result.model,
140
- items: result.items,
141
- sessions: result.sessions,
142
- usage: result.usage,
143
- latencyMs: result.latencyMs,
144
- };
145
- }
146
- /**
147
- * One extraction, retried on a reply the parser rejects; any other failure is thrown. When the last attempt, too, was
148
- * answered with a reply that is not a table, that failure is remembered under `key`; an error before any reply never is.
149
- */
150
- async function extractEvidence(extractor, key, query, sessions, reason, parseRetries) {
151
- const maxAttempts = Math.max(1, parseRetries ?? 3);
152
- let lastError;
153
- for (let attempt = 1; attempt <= maxAttempts; attempt++) {
154
- try {
155
- return evidenceOk(reason, await extractor.extract(query, sessions));
156
- }
157
- catch (err) {
158
- // The workspace's daily allowance is spent: no call was made, and the search goes on without the table.
159
- if (err instanceof optin_1.EvidenceCapError)
160
- return { status: 'skipped', reason: 'daily-cap', model: extractor.model };
161
- lastError = err instanceof Error ? err : new Error(String(err));
162
- const parseFailure = /JSON|items list|non-empty item|must be an object/i.test(lastError.message);
163
- if (!parseFailure || attempt === maxAttempts)
164
- break;
165
- }
166
- }
167
- const parseFailure = lastError && /JSON|items list|non-empty item|must be an object/i.test(lastError.message);
168
- if (parseFailure && lastError) {
169
- const error = lastError.message.slice(0, 200);
170
- if (lastError instanceof secondPass_1.EvidenceReplyError)
171
- extractor.rememberFailure(key, error);
172
- return {
173
- status: 'skipped',
174
- reason: 'extractor-parse-error',
175
- model: extractor.model,
176
- error,
177
- };
178
- }
179
- throw lastError ?? new Error('evidence extraction failed');
180
- }
181
- /** H10-v4 fact-row mapping used by the frozen QA facts-table renderer. */
182
- function evidenceItemsToFactRows(items, event = 'second-pass-v4') {
183
- return items.map(item => {
184
- const amount = String(item.amount ?? '').trim();
185
- const match = amount.match(/^\$?([\d,]+(?:\.\d+)?)\s*(.*)$/);
186
- let value = 1;
187
- let unit = amount;
188
- if (amount && match) {
189
- value = match[1].replace(/,/g, '');
190
- unit = match[2] ?? '';
191
- }
192
- return {
193
- value,
194
- unit,
195
- verb: item.action ?? '',
196
- object: item.item ?? '',
197
- event,
198
- confirm: 1,
199
- dates: [item.date ?? ''],
200
- };
201
- });
202
- }
203
- function renderEvidenceTable(items) {
204
- const lines = items.map(item => {
205
- const action = item.action?.trim() ? ` | ${item.action.trim()}` : '';
206
- const amount = item.amount?.trim() ? ` | ${item.amount.trim()}` : '';
207
- const date = item.date?.trim() ? ` | ${item.date.trim()}` : '';
208
- return `- ${item.item}${action}${amount}${date}`;
209
- });
210
- return [
211
- 'User-asserted items extracted from full source notes (with dates):',
212
- ...lines,
213
- 'Use this table when relevant; cite only values present here. If the table does not contain the value needed to answer, use the excerpts instead.',
214
- ].join('\n');
215
- }
216
- function evidenceItemsToHit(items) {
217
- return {
218
- notePath: exports.EVIDENCE_HIT_PATH,
219
- chunkId: -1,
220
- headingPath: 'Evidence',
221
- text: renderEvidenceTable(items),
222
- score: 1,
223
- reason: 'evidence',
224
- };
225
- }
226
- function prependEvidenceHit(hits, attachment) {
227
- if (attachment.status !== 'ok' || !attachment.items?.length)
228
- return hits;
229
- if (hits.some(hit => hit.notePath === exports.EVIDENCE_HIT_PATH && hit.reason === 'evidence'))
230
- return hits;
231
- return [evidenceItemsToHit(attachment.items), ...hits];
232
- }
233
- async function attachSearchEvidence(args) {
234
- if (!(args.enabled ?? searchEvidenceEnabled())) {
235
- return { hits: args.hits, evidence: { status: 'skipped', reason: 'flag-off' } };
236
- }
237
- try {
238
- const evidence = await collectEvidence({
239
- transformBody: args.transformBody,
240
- query: args.query,
241
- hits: args.hits,
242
- vaultPath: args.vaultPath,
243
- options: args.options,
244
- extractor: args.extractor,
245
- force: args.force,
246
- });
247
- return { hits: prependEvidenceHit(args.hits, evidence), evidence };
248
- }
249
- catch (err) {
250
- // Evidence is optional augmentation. Provider/network/cache failures must
251
- // never take down the base search path, but they are said, with the
252
- // extractor that failed, never left as a silently missing table (#222).
253
- const message = err instanceof Error ? err.message : String(err);
254
- return {
255
- hits: args.hits,
256
- evidence: {
257
- status: 'skipped',
258
- reason: 'extractor-error',
259
- ...(args.extractor ? { model: args.extractor.model } : {}),
260
- error: message.slice(0, 200),
261
- },
262
- };
263
- }
264
- }
@@ -1,147 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.EvidenceFlights = exports.EVIDENCE_BACKGROUND_QUEUE = exports.EVIDENCE_BACKGROUND_CONCURRENCY = exports.EVIDENCE_WAIT_MS_MAX = exports.EVIDENCE_WAIT_MS_DEFAULT = void 0;
4
- exports.evidenceAsyncEnabled = evidenceAsyncEnabled;
5
- exports.resolveEvidenceWaitMs = resolveEvidenceWaitMs;
6
- exports.resolveEvidenceBackground = resolveEvidenceBackground;
7
- exports.sharedEvidenceFlights = sharedEvidenceFlights;
8
- exports.waitAtMost = waitAtMost;
9
- /**
10
- * Bounded-wait evidence for get_context and search_memory (RENOR_EVIDENCE_ASYNC, default OFF).
11
- *
12
- * The evidence second pass runs inside the read tools, so a search waits for the extractor: about 1-4 s with today's
13
- * model, 5-27 s on average with the better ones measured on 2026-10-03 (p95 up to 76 s). With this flag off nothing
14
- * here runs and every search waits, as before. With it on, a search waits at most RENOR_EVIDENCE_WAIT_MS:
15
- *
16
- * - a table already in the evidence cache (keyed on model + exact prompt) is returned at once;
17
- * - otherwise the extraction starts, and if it does not finish within the wait the response has no table and says
18
- * so (`state: "pending"`); the extraction carries on in the background and its table lands in the cache, so the
19
- * same query over the same notes returns it;
20
- * - one extraction per prompt (single-flight): a search for a prompt already being extracted joins that extraction;
21
- * - at most EVIDENCE_BACKGROUND_CONCURRENCY extractions run and EVIDENCE_BACKGROUND_QUEUE wait; a search past that
22
- * starts nothing and says so (`state: "busy"`).
23
- *
24
- * Only the MVP read tools read the flag (resolveEngineEvidence). The CLI, which is the benchmark's entry point, the
25
- * HTTP API and the stdio MCP server never do, so no benchmark run can change behaviour because of it.
26
- */
27
- exports.EVIDENCE_WAIT_MS_DEFAULT = 2_000;
28
- exports.EVIDENCE_WAIT_MS_MAX = 60_000;
29
- /** Extractions running at once, per process (one process serves one workspace). */
30
- exports.EVIDENCE_BACKGROUND_CONCURRENCY = 2;
31
- /** Extractions waiting for a free slot, per process. */
32
- exports.EVIDENCE_BACKGROUND_QUEUE = 16;
33
- /** RENOR_EVIDENCE_ASYNC: on only for 1 / true / yes / on; unset and anything else is off. */
34
- function evidenceAsyncEnabled(env = process.env) {
35
- const raw = (env.RENOR_EVIDENCE_ASYNC ?? '').trim().toLowerCase();
36
- return raw === '1' || raw === 'true' || raw === 'yes' || raw === 'on';
37
- }
38
- /** RENOR_EVIDENCE_WAIT_MS: whole milliseconds, 0 to 60,000; unset or unreadable is the default. */
39
- function resolveEvidenceWaitMs(env = process.env) {
40
- const raw = (env.RENOR_EVIDENCE_WAIT_MS ?? '').trim();
41
- const ms = raw === '' ? Number.NaN : Number(raw);
42
- if (!Number.isFinite(ms) || ms < 0)
43
- return exports.EVIDENCE_WAIT_MS_DEFAULT;
44
- return Math.min(Math.floor(ms), exports.EVIDENCE_WAIT_MS_MAX);
45
- }
46
- /** The bounded-wait settings when the flag is on, else undefined: a search then waits for the extractor, as before. */
47
- function resolveEvidenceBackground(env = process.env) {
48
- if (!evidenceAsyncEnabled(env))
49
- return undefined;
50
- return { waitMs: resolveEvidenceWaitMs(env), flights: sharedEvidenceFlights() };
51
- }
52
- /**
53
- * The extractions in the air, one per key, with at most `concurrency` running and `queue` waiting.
54
- *
55
- * Every search of a process must share one of these: the MVP server builds a fresh tool set (and extractor) for each
56
- * SSE session, so flights kept per extractor would neither join a second session's identical search nor bound the
57
- * process's spend.
58
- */
59
- class EvidenceFlights {
60
- concurrency;
61
- queue;
62
- inFlight = new Map();
63
- waiting = [];
64
- running = 0;
65
- /**
66
- * started: extractions begun; joined: searches that shared one; refused: searches turned away by a full queue;
67
- * failed: extractions that ended without a table.
68
- */
69
- counts = { started: 0, joined: 0, refused: 0, failed: 0 };
70
- constructor(limits = {}) {
71
- this.concurrency = Math.max(1, limits.concurrency ?? exports.EVIDENCE_BACKGROUND_CONCURRENCY);
72
- this.queue = Math.max(0, limits.queue ?? exports.EVIDENCE_BACKGROUND_QUEUE);
73
- }
74
- /**
75
- * The one extraction of `key`: the flight already in the air, or a new one that `run` performs now or once a slot is
76
- * free. Null when the queue is full, and then nothing is started. The key is deleted when the flight ends, after
77
- * `run` has written the cache, so a later search finds the table there instead of starting again.
78
- */
79
- join(key, run) {
80
- const current = this.inFlight.get(key);
81
- if (current) {
82
- this.counts.joined += 1;
83
- return current;
84
- }
85
- if (this.running >= this.concurrency && this.waiting.length >= this.queue) {
86
- this.counts.refused += 1;
87
- return null;
88
- }
89
- const slot = new Promise(resolve => {
90
- if (this.running < this.concurrency) {
91
- this.running += 1;
92
- resolve();
93
- }
94
- else {
95
- this.waiting.push(() => {
96
- this.running += 1;
97
- resolve();
98
- });
99
- }
100
- });
101
- const flight = slot.then(run).finally(() => {
102
- this.inFlight.delete(key);
103
- this.running -= 1;
104
- this.waiting.shift()?.();
105
- });
106
- // A flight outlives the searches that started it, so its end is observed here: when it fails after they stopped
107
- // waiting nobody else holds the promise, and a rejection must never surface as an unhandled one; and an operator
108
- // reading the log is the only one left to learn that it failed. The reason code only: never the query or the
109
- // provider's message, which may quote the user's notes.
110
- flight.then(outcome => {
111
- if (outcome.status !== 'ok')
112
- this.ended(outcome.reason ?? 'skipped');
113
- }, () => this.ended('extractor-error'));
114
- this.inFlight.set(key, flight);
115
- this.counts.started += 1;
116
- return flight;
117
- }
118
- ended(reason) {
119
- this.counts.failed += 1;
120
- process.stderr.write(`[evidence] a background extraction ended without a table (${reason})\n`);
121
- }
122
- /** Resolves once nothing is running or waiting (tests, benchmarks). */
123
- async idle() {
124
- while (this.inFlight.size > 0)
125
- await Promise.allSettled([...this.inFlight.values()]);
126
- }
127
- }
128
- exports.EvidenceFlights = EvidenceFlights;
129
- let shared;
130
- /** The process's flights, created on first use. */
131
- function sharedEvidenceFlights() {
132
- shared ??= new EvidenceFlights();
133
- return shared;
134
- }
135
- /** `promise`'s value if it settles within `ms`, else undefined. A later rejection is not this caller's to handle. */
136
- function waitAtMost(promise, ms) {
137
- return new Promise((resolve, reject) => {
138
- const timer = setTimeout(() => resolve(undefined), ms);
139
- promise.then(value => {
140
- clearTimeout(timer);
141
- resolve({ value });
142
- }, err => {
143
- clearTimeout(timer);
144
- reject(err);
145
- });
146
- });
147
- }