renor-engine 1.2.0 → 1.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (216) hide show
  1. package/README.md +34 -2
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/bin/renor-engine.cjs +38 -0
  4. package/dist/BUILD.json +6 -0
  5. package/dist/cli.js +1674 -516
  6. package/package.json +17 -80
  7. package/dist/.tsbuildinfo +0 -1
  8. package/dist/api/apiAuth.js +0 -43
  9. package/dist/api/server.js +0 -174
  10. package/dist/buildIdentity.js +0 -149
  11. package/dist/callPhases.js +0 -82
  12. package/dist/cli-extractors.js +0 -46
  13. package/dist/config.js +0 -65
  14. package/dist/context/composer.js +0 -320
  15. package/dist/context/readHonesty.js +0 -478
  16. package/dist/context/rowStore.js +0 -238
  17. package/dist/conversation/formatter.js +0 -81
  18. package/dist/conversation/parser.js +0 -193
  19. package/dist/conversation/types.js +0 -2
  20. package/dist/conversation/writer.js +0 -42
  21. package/dist/db/client.js +0 -149
  22. package/dist/db/handoffTable.js +0 -31
  23. package/dist/db/indexSalvage.js +0 -101
  24. package/dist/db/migrations.js +0 -670
  25. package/dist/db/readCache.js +0 -109
  26. package/dist/db/schema.js +0 -5
  27. package/dist/db/sqlite.js +0 -396
  28. package/dist/db/statements.js +0 -89
  29. package/dist/digests/digestCache.js +0 -146
  30. package/dist/digests/digests.js +0 -597
  31. package/dist/digests/worker.js +0 -350
  32. package/dist/embeddings/caching.js +0 -337
  33. package/dist/embeddings/coalescing.js +0 -177
  34. package/dist/embeddings/interactive.js +0 -103
  35. package/dist/embeddings/local.js +0 -70
  36. package/dist/embeddings/openai.js +0 -106
  37. package/dist/embeddings/rebuild.js +0 -166
  38. package/dist/embeddings/simdScan.js +0 -260
  39. package/dist/embeddings/space.js +0 -21
  40. package/dist/embeddings/types.js +0 -14
  41. package/dist/embeddings/vectorIndex.js +0 -674
  42. package/dist/embeddings/vertex.js +0 -334
  43. package/dist/eval/artifacts.js +0 -42
  44. package/dist/eval/cacheReport.js +0 -146
  45. package/dist/eval/comparison.js +0 -182
  46. package/dist/eval/context700-candidate.js +0 -145
  47. package/dist/eval/evalCache.js +0 -568
  48. package/dist/eval/hashEmbedding.js +0 -35
  49. package/dist/eval/indexSnapshots.js +0 -549
  50. package/dist/eval/longmemeval.js +0 -56
  51. package/dist/eval/metrics.js +0 -181
  52. package/dist/eval/pageDump.js +0 -67
  53. package/dist/eval/renorbenchBaseline.js +0 -486
  54. package/dist/eval/renorbenchDocuments.js +0 -134
  55. package/dist/eval/renorbenchProduct.js +0 -182
  56. package/dist/eval/runLedger.js +0 -279
  57. package/dist/eval/runManifest.js +0 -485
  58. package/dist/eval/scoring.js +0 -197
  59. package/dist/eval/textTokens.js +0 -17
  60. package/dist/evidence/attach.js +0 -264
  61. package/dist/evidence/background.js +0 -147
  62. package/dist/evidence/evidenceCache.js +0 -182
  63. package/dist/evidence/gate.js +0 -37
  64. package/dist/evidence/keyPool.js +0 -85
  65. package/dist/evidence/ollamaChat.js +0 -84
  66. package/dist/evidence/optin.js +0 -165
  67. package/dist/evidence/options.js +0 -151
  68. package/dist/evidence/secondPass.js +0 -294
  69. package/dist/extraction/classifyCapture.js +0 -34
  70. package/dist/extraction/decision-model.js +0 -174
  71. package/dist/extraction/eventExtractor.js +0 -113
  72. package/dist/extraction/extractionClient.js +0 -122
  73. package/dist/extraction/factExtractor.js +0 -46
  74. package/dist/extraction/factsWritePath.js +0 -179
  75. package/dist/extraction/jevDecisionModel.js +0 -307
  76. package/dist/extraction/preferenceExtractor.js +0 -150
  77. package/dist/extraction/secretGuard.js +0 -180
  78. package/dist/extraction/secretScanner.js +0 -665
  79. package/dist/extraction/typedFacts.js +0 -224
  80. package/dist/extraction/writePathAdjudication.js +0 -110
  81. package/dist/importing/apply.js +0 -672
  82. package/dist/importing/bundle-write.js +0 -80
  83. package/dist/importing/cli.js +0 -408
  84. package/dist/importing/existing.js +0 -51
  85. package/dist/importing/facts.js +0 -121
  86. package/dist/importing/forgetImported.js +0 -79
  87. package/dist/importing/hosted.js +0 -467
  88. package/dist/importing/journalLog.js +0 -96
  89. package/dist/importing/markdown.js +0 -98
  90. package/dist/importing/plan.js +0 -201
  91. package/dist/importing/secretScreen.js +0 -113
  92. package/dist/importing/serialize.js +0 -188
  93. package/dist/importing/source.js +0 -463
  94. package/dist/importing/types.js +0 -6
  95. package/dist/indexer/chunker.js +0 -319
  96. package/dist/indexer/dailyEntries.js +0 -63
  97. package/dist/indexer/hasher.js +0 -10
  98. package/dist/indexer/laneRuns.js +0 -48
  99. package/dist/indexer/o200k.js +0 -481
  100. package/dist/indexer/outsideRoot.js +0 -77
  101. package/dist/indexer/parser.js +0 -170
  102. package/dist/indexer/pipeline.js +0 -988
  103. package/dist/indexer/staleness.js +0 -201
  104. package/dist/indexer/tokenizer.js +0 -77
  105. package/dist/indexer/walker.js +0 -170
  106. package/dist/learning/cli.js +0 -113
  107. package/dist/learning/schema.js +0 -144
  108. package/dist/learning/settings.js +0 -45
  109. package/dist/learning/store.js +0 -267
  110. package/dist/learning/tap.js +0 -401
  111. package/dist/mcp/hive.js +0 -88
  112. package/dist/mcp/requestErrors.js +0 -47
  113. package/dist/mcp/server.js +0 -426
  114. package/dist/memory/agentLease.js +0 -43
  115. package/dist/memory/confinedWrite.js +0 -315
  116. package/dist/memory/controls.js +0 -73
  117. package/dist/memory/dailyNote.js +0 -265
  118. package/dist/memory/dreamer/adjudicate.js +0 -215
  119. package/dist/memory/dreamer/apply.js +0 -120
  120. package/dist/memory/dreamer/budget.js +0 -64
  121. package/dist/memory/dreamer/candidates.js +0 -149
  122. package/dist/memory/dreamer/collection.js +0 -104
  123. package/dist/memory/dreamer/dreamer.js +0 -352
  124. package/dist/memory/dreamer/review.js +0 -111
  125. package/dist/memory/durableAppend.js +0 -57
  126. package/dist/memory/factConflicts.js +0 -205
  127. package/dist/memory/factTerms.js +0 -321
  128. package/dist/memory/handoff.js +0 -363
  129. package/dist/memory/hiveNameHeader.cjs +0 -10
  130. package/dist/memory/inventory.js +0 -55
  131. package/dist/memory/invisibleText.js +0 -20
  132. package/dist/memory/lifecycle.js +0 -2014
  133. package/dist/memory/limits.js +0 -13
  134. package/dist/memory/requests.js +0 -183
  135. package/dist/memory/sourceName.js +0 -276
  136. package/dist/memory/statusFacts.js +0 -61
  137. package/dist/memory/whatsNew.js +0 -158
  138. package/dist/memory/writer.js +0 -176
  139. package/dist/memory/writerConflicts.js +0 -85
  140. package/dist/memorymap/memoryMap.js +0 -1594
  141. package/dist/memorymap/replacedJournal.js +0 -58
  142. package/dist/memorymap/replacements.js +0 -410
  143. package/dist/mvp/activity.js +0 -86
  144. package/dist/mvp/agentMemory.js +0 -1803
  145. package/dist/mvp/answerCheck.js +0 -211
  146. package/dist/mvp/auth.js +0 -531
  147. package/dist/mvp/cli.js +0 -86
  148. package/dist/mvp/client-store.js +0 -462
  149. package/dist/mvp/compileCache.js +0 -242
  150. package/dist/mvp/compileSeed.js +0 -33
  151. package/dist/mvp/config.js +0 -292
  152. package/dist/mvp/connectionStatus.js +0 -80
  153. package/dist/mvp/contextDistinct.js +0 -53
  154. package/dist/mvp/contextProfile.js +0 -87
  155. package/dist/mvp/contextRelevance.js +0 -413
  156. package/dist/mvp/directorySurface.js +0 -68
  157. package/dist/mvp/engine.js +0 -1078
  158. package/dist/mvp/exportAll.js +0 -393
  159. package/dist/mvp/factLanePack.js +0 -77
  160. package/dist/mvp/factMarkers.js +0 -65
  161. package/dist/mvp/factsLane.js +0 -203
  162. package/dist/mvp/hiddenSpans.js +0 -585
  163. package/dist/mvp/history.js +0 -414
  164. package/dist/mvp/hitDates.js +0 -31
  165. package/dist/mvp/hitFloor.js +0 -36
  166. package/dist/mvp/identity.js +0 -18
  167. package/dist/mvp/idle.js +0 -113
  168. package/dist/mvp/log.js +0 -118
  169. package/dist/mvp/memoryDir.js +0 -98
  170. package/dist/mvp/posthogMcp.js +0 -210
  171. package/dist/mvp/readScope.js +0 -104
  172. package/dist/mvp/readUsefulness.js +0 -222
  173. package/dist/mvp/sanitize.js +0 -48
  174. package/dist/mvp/schemaRejections.js +0 -91
  175. package/dist/mvp/scope.js +0 -127
  176. package/dist/mvp/server.js +0 -2712
  177. package/dist/mvp/serverCard.js +0 -38
  178. package/dist/mvp/storedSize.js +0 -139
  179. package/dist/mvp/telemetry.js +0 -474
  180. package/dist/net/poolFetch.js +0 -174
  181. package/dist/recall/actionRecall.js +0 -282
  182. package/dist/recall/autoRecall.js +0 -447
  183. package/dist/recall/searchRecall.js +0 -189
  184. package/dist/retrieval/adviceLines.js +0 -195
  185. package/dist/retrieval/adviceQuestion.js +0 -44
  186. package/dist/retrieval/conversationShape.js +0 -44
  187. package/dist/retrieval/dailyEntryTime.js +0 -55
  188. package/dist/retrieval/entityLookup.js +0 -162
  189. package/dist/retrieval/factsFirst.js +0 -188
  190. package/dist/retrieval/keyword.js +0 -84
  191. package/dist/retrieval/ledgerEventTime.js +0 -51
  192. package/dist/retrieval/ledgerLiveness.js +0 -83
  193. package/dist/retrieval/ledgerPresentation.js +0 -161
  194. package/dist/retrieval/noteIdentity.js +0 -57
  195. package/dist/retrieval/orchestrator.js +0 -1224
  196. package/dist/retrieval/plainWording.js +0 -236
  197. package/dist/retrieval/queryHelpers.js +0 -142
  198. package/dist/retrieval/questionSpan.js +0 -475
  199. package/dist/retrieval/rankingTrace.js +0 -184
  200. package/dist/retrieval/readerEvents.js +0 -690
  201. package/dist/retrieval/readerLayout.js +0 -422
  202. package/dist/retrieval/relativeTimeHints.js +0 -113
  203. package/dist/retrieval/semantic.js +0 -67
  204. package/dist/router/routerCache.js +0 -106
  205. package/dist/router/sessionRouter.js +0 -726
  206. package/dist/sharing/cli.js +0 -181
  207. package/dist/sharing/examples.js +0 -136
  208. package/dist/sharing/fields.js +0 -225
  209. package/dist/sharing/loop.js +0 -80
  210. package/dist/sharing/project.js +0 -327
  211. package/dist/sharing/send.js +0 -466
  212. package/dist/sharing/state.js +0 -222
  213. package/dist/sharing/wording.js +0 -48
  214. package/dist/toolRefusal.js +0 -120
  215. package/dist/watcher/watcher.js +0 -77
  216. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,182 +0,0 @@
1
- "use strict";
2
- /**
3
- * Pure comparison policy between two eval runs. Extracted from the CLI
4
- * wrapper so the promotion logic itself lives in the frozen evaluator
5
- * trust domain (its hash is recorded in run manifests) and is unit tested.
6
- */
7
- Object.defineProperty(exports, "__esModule", { value: true });
8
- exports.QUALITY_METRICS = void 0;
9
- exports.compareRuns = compareRuns;
10
- exports.QUALITY_METRICS = [
11
- { key: 'hitAt1', higherBetter: true },
12
- { key: 'hitAt5', higherBetter: true },
13
- { key: 'recallAt5', higherBetter: true },
14
- { key: 'recallAllAt10', higherBetter: true },
15
- { key: 'mrr', higherBetter: true },
16
- { key: 'ndcgAt5', higherBetter: true },
17
- { key: 'contradictionFirstRate', higherBetter: false },
18
- { key: 'forbiddenRetrievalRate', higherBetter: false },
19
- { key: 'abstentionEmptyRate', higherBetter: true },
20
- ];
21
- const EFFICIENCY_METRICS = ['meanLatencyMs', 'p50LatencyMs', 'p95LatencyMs', 'meanContextTokens'];
22
- const MATERIAL_EFFICIENCY_METRICS = ['p95LatencyMs', 'meanContextTokens'];
23
- function same(a, b) {
24
- return JSON.stringify(a) === JSON.stringify(b);
25
- }
26
- function num(map, key) {
27
- const value = map[key];
28
- return typeof value === 'number' && Number.isFinite(value) ? value : null;
29
- }
30
- function delta(candidate, baseline) {
31
- return candidate === null || baseline === null ? null : Math.round((candidate - baseline) * 10000) / 10000;
32
- }
33
- /** Relative improvement of candidate over baseline; null when not computable. */
34
- function relativeGain(baseline, candidate) {
35
- if (baseline === null || candidate === null || baseline <= 0)
36
- return null;
37
- return (baseline - candidate) / baseline;
38
- }
39
- function compareRuns(baseline, candidate, options = {}) {
40
- const tolerance = options.maxQualityRegression ?? 0;
41
- const metricList = options.metrics ?? exports.QUALITY_METRICS;
42
- const minEfficiencyGain = options.minEfficiencyGain ?? 0.05;
43
- if (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1) {
44
- throw new Error('maxQualityRegression must be between 0 and 1');
45
- }
46
- if (!Number.isFinite(minEfficiencyGain) || minEfficiencyGain < 0 || minEfficiencyGain > 1) {
47
- throw new Error('minEfficiencyGain must be between 0 and 1');
48
- }
49
- const invariantChecks = {
50
- benchmark: same(baseline.manifest.benchmark, candidate.manifest.benchmark),
51
- cases: same(baseline.manifest.casesSha256, candidate.manifest.casesSha256),
52
- fixture: same(baseline.manifest.fixtureSha256, candidate.manifest.fixtureSha256),
53
- topK: same(baseline.manifest.topK, candidate.manifest.topK),
54
- provider: same(baseline.manifest.provider, candidate.manifest.provider),
55
- chunking: same(baseline.manifest.chunking, candidate.manifest.chunking),
56
- evaluator: same(baseline.manifest.evaluatorSha256, candidate.manifest.evaluatorSha256),
57
- cleanTree: baseline.manifest.gitDirty === false && candidate.manifest.gitDirty === false,
58
- qualityEligible: baseline.manifest.qualityEligible === true && candidate.manifest.qualityEligible === true,
59
- };
60
- const reasons = [];
61
- const datasetCompatible = invariantChecks.benchmark && invariantChecks.cases && invariantChecks.fixture;
62
- if (!datasetCompatible) {
63
- throw new Error(`Cannot compare runs from different benchmark data: ${JSON.stringify(invariantChecks)}`);
64
- }
65
- if (!invariantChecks.evaluator) {
66
- if (!options.allowEvaluatorChange) {
67
- throw new Error('Evaluator changed between baseline and candidate (evaluatorSha256 mismatch). ' +
68
- 'The candidate cannot grade its own exam: re-run the candidate against the frozen evaluator, ' +
69
- 'or pass --allow-evaluator-change to accept a DESCRIPTIVE_ONLY comparison during evaluator development.');
70
- }
71
- reasons.push('Evaluator code changed between runs; measurement meaning differs.');
72
- }
73
- if (!invariantChecks.cleanTree) {
74
- if (!options.allowDirtyTree) {
75
- reasons.push('Working tree was dirty in at least one run; executed code does not match the recorded git SHA.');
76
- }
77
- else {
78
- reasons.push('Dirty working tree tolerated (--allow-dirty-tree).');
79
- }
80
- }
81
- if (!invariantChecks.qualityEligible) {
82
- reasons.push('At least one run used a non-quality provider (e.g. hash embeddings); infrastructure evidence only.');
83
- }
84
- const configCompatible = invariantChecks.topK && invariantChecks.provider && invariantChecks.chunking;
85
- if (!configCompatible) {
86
- reasons.push('Provider, topK, or chunking changed between runs.');
87
- }
88
- const regressionGate = options.regressionGate === true;
89
- const experimentCompatible = Boolean(invariantChecks.evaluator
90
- && (regressionGate || (invariantChecks.cleanTree && invariantChecks.qualityEligible))
91
- && (configCompatible || (!regressionGate && options.allowConfigChange)));
92
- const quality = Object.fromEntries(metricList.map(metric => {
93
- const baselineValue = num(baseline.summary.overall, metric.key);
94
- const candidateValue = num(candidate.summary.overall, metric.key);
95
- return [metric.key, { baseline: baselineValue, candidate: candidateValue, delta: delta(candidateValue, baselineValue) }];
96
- }));
97
- const efficiency = Object.fromEntries(EFFICIENCY_METRICS.map(key => {
98
- const baselineValue = num(baseline.summary.overall, key);
99
- const candidateValue = num(candidate.summary.overall, key);
100
- return [key, { baseline: baselineValue, candidate: candidateValue, delta: delta(candidateValue, baselineValue) }];
101
- }));
102
- let verdict;
103
- if (!experimentCompatible) {
104
- verdict = 'DESCRIPTIVE_ONLY';
105
- }
106
- else {
107
- const hasRegression = metricList.some(metric => {
108
- const { delta: d } = quality[metric.key];
109
- if (typeof d !== 'number')
110
- return false;
111
- return metric.higherBetter ? d < -tolerance : d > tolerance;
112
- });
113
- const hasQualityGain = metricList.some(metric => {
114
- const { delta: d } = quality[metric.key];
115
- if (typeof d !== 'number')
116
- return false;
117
- return metric.higherBetter ? d > tolerance : d < -tolerance;
118
- });
119
- const hasMaterialEfficiencyGain = MATERIAL_EFFICIENCY_METRICS.some(key => {
120
- const gain = relativeGain(efficiency[key].baseline, efficiency[key].candidate);
121
- return gain !== null && gain >= minEfficiencyGain;
122
- });
123
- if (hasRegression)
124
- verdict = 'REGRESSED';
125
- else if (hasQualityGain || hasMaterialEfficiencyGain)
126
- verdict = 'IMPROVED';
127
- else
128
- verdict = 'NEUTRAL';
129
- }
130
- // Category-collapse guard: an aggregate improvement must not hide one
131
- // category regressing beyond tolerance (independent-review finding).
132
- const categoryRegressions = [];
133
- const baselineCategories = baseline.summary.byCategory ?? {};
134
- const candidateCategories = candidate.summary.byCategory ?? {};
135
- // Both sides' categories: a category the baseline has and the candidate lost is a regression too.
136
- for (const name of new Set([...Object.keys(baselineCategories), ...Object.keys(candidateCategories)])) {
137
- const baselineMetrics = baselineCategories[name];
138
- const metrics = candidateCategories[name];
139
- if (!baselineMetrics)
140
- continue; // new category: nothing to regress against
141
- if (!metrics) {
142
- categoryRegressions.push(`${name}.missing`);
143
- continue;
144
- }
145
- for (const metric of metricList) {
146
- const b = num(baselineMetrics, metric.key);
147
- const c = num(metrics, metric.key);
148
- if (b === null || c === null)
149
- continue;
150
- const d = delta(c, b);
151
- if (typeof d === 'number' && (metric.higherBetter ? d < -tolerance : d > tolerance)) {
152
- categoryRegressions.push(`${name}.${metric.key}`);
153
- }
154
- }
155
- }
156
- for (const name of categoryRegressions) {
157
- reasons.push(`Category regression beyond tolerance: ${name}`);
158
- }
159
- if (categoryRegressions.length > 0) {
160
- // A category regression beyond tolerance is a rejection regardless of the
161
- // aggregate verdict: an overall gain must not hide a collapsing category.
162
- verdict = 'REGRESSED';
163
- }
164
- const note = regressionGate
165
- ? 'Regression gate against a committed baseline: this is a no-regression check, never a clean ablation or a promotion.'
166
- : experimentCompatible
167
- ? 'IMPROVED requires no quality regression beyond tolerance and a quality gain or a material (>= threshold) efficiency gain.'
168
- : 'Not a clean ablation: this comparison is descriptive only and cannot support promotion.';
169
- return {
170
- verdict,
171
- cleanAblation: experimentCompatible && !regressionGate,
172
- regressionGate,
173
- invariantChecks,
174
- tolerance,
175
- minEfficiencyGain,
176
- quality,
177
- efficiency,
178
- categoryRegressions,
179
- reasons,
180
- note,
181
- };
182
- }
@@ -1,145 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.canonicalSegment = exports.COMPILER_VERSION = exports.isHeaderOnly = void 0;
4
- exports.compileContext = compileContext;
5
- /**
6
- * Context700 campaign — MUTABLE context compiler.
7
- *
8
- * Campaign contract (docs/agents/autoresearch/context700/program.md):
9
- * - Target: <= 700 mean Renor memory-evidence tokens/query on LongMemEval-S full-500.
10
- * - Frozen quality floor: >= 428/500, no category regression > 2.0 pp vs N22 (430/500).
11
- * - ONLY this file is mutable after harness freeze.
12
- *
13
- * The compiler is gold-blind: it may use the question text/date, retrieval hits,
14
- * and structured extraction tables. It may NOT use gold answers, benchmark
15
- * question IDs (as decision keys), judge outputs, or correctness labels.
16
- *
17
- * v3 (structured-first + date-preserving compaction + recency raw fill):
18
- * Lesson from v1 (31/118 screen KILL): raw-first 55% budget + aggressive
19
- * compaction starved date-bearing stubs and chronicle breadth. v3:
20
- * 1. Query-gated structured rows FIRST (facts cap 12, chronicle cap 10, prefs cap 6).
21
- * 2. Raw excerpts: compact turn markers, PRESERVE Timestamp/date lines and
22
- * header stubs (they carry session dates), recency-ordered like N22.
23
- * 3. Fill remaining budget by rank; no minimum raw count (tables win ties).
24
- */
25
- const textTokens_1 = require("./textTokens");
26
- const canonicalSegment = (notePath) => notePath.replace(/^Sessions\//, '').replace(/-occ\d+\.md$/, '.md').replace(/\.md$/, '');
27
- exports.canonicalSegment = canonicalSegment;
28
- const isHeaderOnly = (text) => !text.includes('### Turn') && text.trim().length < 250;
29
- exports.isHeaderOnly = isHeaderOnly;
30
- exports.COMPILER_VERSION = 'v3-struct-first-date-keeping';
31
- const CONTENT_STOP = new Set('the a an of to for and or in on my i with is are was be that this it at as by from have has had do does did not no you your me we our they their he she his her about into over under then than so if but'.split(' '));
32
- const contentWords = (s) => {
33
- const out = new Set();
34
- for (const w of s.toLowerCase().matchAll(/[a-z0-9]{3,}/g)) {
35
- if (!CONTENT_STOP.has(w[0]))
36
- out.add(w[0]);
37
- }
38
- return out;
39
- };
40
- /** Compact date: "2023/05/30 (Tue) 23:40" -> "2023/05/30". */
41
- const compactDate = (d) => d ? d.replace(/\s*\((?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\).*/, '') : '';
42
- /**
43
- * v3 compaction: shorten turn markers to `[N-role]`, drop long session headers,
44
- * but PRESERVE Timestamp lines (dates are load-bearing for temporal/multi-session).
45
- */
46
- const compactText = (t) => t
47
- .replace(/^### Turn (\d+) — (User|Assistant)\s*\|?\s*/gim, '[$1-$2] ')
48
- .replace(/^Timestamp: (.*)$/gim, '@$1')
49
- .replace(/^# Session .*\n?/gim, '')
50
- .replace(/\n{3,}/g, '\n\n')
51
- .trim();
52
- const gateRows = (rows, qWords, cap) => {
53
- if (!rows || rows.length === 0)
54
- return [];
55
- const scored = [];
56
- for (const row of rows) {
57
- const text = Object.values(row).map(v => (typeof v === 'string' ? v : JSON.stringify(v) ?? '')).join(' ');
58
- let overlap = 0;
59
- for (const w of contentWords(text))
60
- if (qWords.has(w))
61
- overlap++;
62
- if (overlap > 0)
63
- scored.push({ score: overlap, row });
64
- }
65
- scored.sort((a, b) => b.score - a.score);
66
- return scored.slice(0, cap);
67
- };
68
- function compileContext(args) {
69
- const { question, hits, tables, dateMap, maxContextTokens } = args;
70
- const qWords = contentWords(question.question);
71
- const parts = { rawTokens: 0, factsTokens: 0, chronicleTokens: 0, prefsTokens: 0 };
72
- const excerpts = [];
73
- // --- Structured rows first (gated) ---
74
- let tokens = 0;
75
- let remaining = maxContextTokens;
76
- const factScored = gateRows(tables.facts ?? [], qWords, 12);
77
- if (factScored.length) {
78
- const lines = factScored.map(({ row: f }) => {
79
- const co = f.coexist ? ` (also: ${f.coexist})` : '';
80
- return `- ${f.value} ${f.unit ?? ''} ${f.object ?? ''}${f.event ? ` — ${f.event}` : ''}${co}`.replace(/\s+/g, ' ').trim();
81
- });
82
- const block = `Numeric facts:\n${lines.join('\n')}\nCite only values present here; otherwise use excerpts.`;
83
- const blockTokens = (0, textTokens_1.countTextTokens)(block);
84
- if (blockTokens <= remaining) {
85
- excerpts.push({ text: block, date: null });
86
- tokens += blockTokens;
87
- parts.factsTokens = blockTokens;
88
- remaining -= blockTokens;
89
- }
90
- }
91
- const chronScored = gateRows(tables.chronicle ?? [], qWords, 10);
92
- if (chronScored.length) {
93
- const lines = chronScored.map(({ row: e }) => `- ${e.when ?? ''}: ${e.event ?? ''}`.replace(/\s+/g, ' ').trim());
94
- const block = `Events:\n${lines.join('\n')}\nCite only events present here; otherwise use excerpts.`;
95
- const blockTokens = (0, textTokens_1.countTextTokens)(block);
96
- if (blockTokens <= remaining) {
97
- excerpts.push({ text: block, date: null });
98
- tokens += blockTokens;
99
- parts.chronicleTokens = blockTokens;
100
- remaining -= blockTokens;
101
- }
102
- }
103
- const prefScored = gateRows(tables.prefs ?? [], qWords, 6);
104
- if (prefScored.length) {
105
- const lines = prefScored.map(({ row: p }) => `- ${p.statement ?? ''}`.replace(/\s+/g, ' ').trim());
106
- const block = `Standing preferences:\n${lines.join('\n')}\nApply matching entries; otherwise use excerpts.`;
107
- const blockTokens = (0, textTokens_1.countTextTokens)(block);
108
- if (blockTokens <= remaining) {
109
- excerpts.push({ text: block, date: null });
110
- tokens += blockTokens;
111
- parts.prefsTokens = blockTokens;
112
- remaining -= blockTokens;
113
- }
114
- }
115
- const raws = [];
116
- const seen = new Set();
117
- for (const hit of args.hits) {
118
- const text = compactText((hit.text ?? '').trim());
119
- if (!text || seen.has(hit.text ?? ''))
120
- continue;
121
- const sessionDate = dateMap?.get(canonicalSegment(hit.notePath)) ?? null;
122
- seen.add(hit.text ?? '');
123
- raws.push({ text, date: sessionDate });
124
- }
125
- // N22 recency ordering (newest-first) before selection.
126
- if (args.recencyOrder) {
127
- raws.sort((a, b) => (b.date ?? '').localeCompare(a.date ?? ''));
128
- }
129
- for (const raw of raws) {
130
- const dateMark = raw.date ? `(${compactDate(raw.date)}) ` : '';
131
- const candidate = `${dateMark}${raw.text}`;
132
- const candidateTokens = (0, textTokens_1.countTextTokens)(candidate);
133
- if (tokens + candidateTokens > maxContextTokens)
134
- break;
135
- tokens += candidateTokens;
136
- excerpts.push({ text: candidate, date: raw.date });
137
- }
138
- parts.rawTokens = tokens - parts.factsTokens - parts.chronicleTokens - parts.prefsTokens;
139
- return {
140
- context: excerpts.map(excerpt => excerpt.text).join('\n\n'),
141
- tokens,
142
- parts,
143
- diagnostics: { compilerVersion: args.compilerVersion },
144
- };
145
- }