renor-engine 1.2.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. package/README.md +21 -1
  2. package/THIRD-PARTY-NOTICES +2661 -0
  3. package/dist/BUILD.json +6 -0
  4. package/dist/cli.js +1656 -516
  5. package/package.json +16 -80
  6. package/dist/.tsbuildinfo +0 -1
  7. package/dist/api/apiAuth.js +0 -43
  8. package/dist/api/server.js +0 -174
  9. package/dist/buildIdentity.js +0 -149
  10. package/dist/callPhases.js +0 -82
  11. package/dist/cli-extractors.js +0 -46
  12. package/dist/config.js +0 -65
  13. package/dist/context/composer.js +0 -320
  14. package/dist/context/readHonesty.js +0 -478
  15. package/dist/context/rowStore.js +0 -238
  16. package/dist/conversation/formatter.js +0 -81
  17. package/dist/conversation/parser.js +0 -193
  18. package/dist/conversation/types.js +0 -2
  19. package/dist/conversation/writer.js +0 -42
  20. package/dist/db/client.js +0 -149
  21. package/dist/db/handoffTable.js +0 -31
  22. package/dist/db/indexSalvage.js +0 -101
  23. package/dist/db/migrations.js +0 -670
  24. package/dist/db/readCache.js +0 -109
  25. package/dist/db/schema.js +0 -5
  26. package/dist/db/sqlite.js +0 -396
  27. package/dist/db/statements.js +0 -89
  28. package/dist/digests/digestCache.js +0 -146
  29. package/dist/digests/digests.js +0 -597
  30. package/dist/digests/worker.js +0 -350
  31. package/dist/embeddings/caching.js +0 -337
  32. package/dist/embeddings/coalescing.js +0 -177
  33. package/dist/embeddings/interactive.js +0 -103
  34. package/dist/embeddings/local.js +0 -70
  35. package/dist/embeddings/openai.js +0 -106
  36. package/dist/embeddings/rebuild.js +0 -166
  37. package/dist/embeddings/simdScan.js +0 -260
  38. package/dist/embeddings/space.js +0 -21
  39. package/dist/embeddings/types.js +0 -14
  40. package/dist/embeddings/vectorIndex.js +0 -674
  41. package/dist/embeddings/vertex.js +0 -334
  42. package/dist/eval/artifacts.js +0 -42
  43. package/dist/eval/cacheReport.js +0 -146
  44. package/dist/eval/comparison.js +0 -182
  45. package/dist/eval/context700-candidate.js +0 -145
  46. package/dist/eval/evalCache.js +0 -568
  47. package/dist/eval/hashEmbedding.js +0 -35
  48. package/dist/eval/indexSnapshots.js +0 -549
  49. package/dist/eval/longmemeval.js +0 -56
  50. package/dist/eval/metrics.js +0 -181
  51. package/dist/eval/pageDump.js +0 -67
  52. package/dist/eval/renorbenchBaseline.js +0 -486
  53. package/dist/eval/renorbenchDocuments.js +0 -134
  54. package/dist/eval/renorbenchProduct.js +0 -182
  55. package/dist/eval/runLedger.js +0 -279
  56. package/dist/eval/runManifest.js +0 -485
  57. package/dist/eval/scoring.js +0 -197
  58. package/dist/eval/textTokens.js +0 -17
  59. package/dist/evidence/attach.js +0 -264
  60. package/dist/evidence/background.js +0 -147
  61. package/dist/evidence/evidenceCache.js +0 -182
  62. package/dist/evidence/gate.js +0 -37
  63. package/dist/evidence/keyPool.js +0 -85
  64. package/dist/evidence/ollamaChat.js +0 -84
  65. package/dist/evidence/optin.js +0 -165
  66. package/dist/evidence/options.js +0 -151
  67. package/dist/evidence/secondPass.js +0 -294
  68. package/dist/extraction/classifyCapture.js +0 -34
  69. package/dist/extraction/decision-model.js +0 -174
  70. package/dist/extraction/eventExtractor.js +0 -113
  71. package/dist/extraction/extractionClient.js +0 -122
  72. package/dist/extraction/factExtractor.js +0 -46
  73. package/dist/extraction/factsWritePath.js +0 -179
  74. package/dist/extraction/jevDecisionModel.js +0 -307
  75. package/dist/extraction/preferenceExtractor.js +0 -150
  76. package/dist/extraction/secretGuard.js +0 -180
  77. package/dist/extraction/secretScanner.js +0 -665
  78. package/dist/extraction/typedFacts.js +0 -224
  79. package/dist/extraction/writePathAdjudication.js +0 -110
  80. package/dist/importing/apply.js +0 -672
  81. package/dist/importing/bundle-write.js +0 -80
  82. package/dist/importing/cli.js +0 -408
  83. package/dist/importing/existing.js +0 -51
  84. package/dist/importing/facts.js +0 -121
  85. package/dist/importing/forgetImported.js +0 -79
  86. package/dist/importing/hosted.js +0 -467
  87. package/dist/importing/journalLog.js +0 -96
  88. package/dist/importing/markdown.js +0 -98
  89. package/dist/importing/plan.js +0 -201
  90. package/dist/importing/secretScreen.js +0 -113
  91. package/dist/importing/serialize.js +0 -188
  92. package/dist/importing/source.js +0 -463
  93. package/dist/importing/types.js +0 -6
  94. package/dist/indexer/chunker.js +0 -319
  95. package/dist/indexer/dailyEntries.js +0 -63
  96. package/dist/indexer/hasher.js +0 -10
  97. package/dist/indexer/laneRuns.js +0 -48
  98. package/dist/indexer/o200k.js +0 -481
  99. package/dist/indexer/outsideRoot.js +0 -77
  100. package/dist/indexer/parser.js +0 -170
  101. package/dist/indexer/pipeline.js +0 -988
  102. package/dist/indexer/staleness.js +0 -201
  103. package/dist/indexer/tokenizer.js +0 -77
  104. package/dist/indexer/walker.js +0 -170
  105. package/dist/learning/cli.js +0 -113
  106. package/dist/learning/schema.js +0 -144
  107. package/dist/learning/settings.js +0 -45
  108. package/dist/learning/store.js +0 -267
  109. package/dist/learning/tap.js +0 -401
  110. package/dist/mcp/hive.js +0 -88
  111. package/dist/mcp/requestErrors.js +0 -47
  112. package/dist/mcp/server.js +0 -426
  113. package/dist/memory/agentLease.js +0 -43
  114. package/dist/memory/confinedWrite.js +0 -315
  115. package/dist/memory/controls.js +0 -73
  116. package/dist/memory/dailyNote.js +0 -265
  117. package/dist/memory/dreamer/adjudicate.js +0 -215
  118. package/dist/memory/dreamer/apply.js +0 -120
  119. package/dist/memory/dreamer/budget.js +0 -64
  120. package/dist/memory/dreamer/candidates.js +0 -149
  121. package/dist/memory/dreamer/collection.js +0 -104
  122. package/dist/memory/dreamer/dreamer.js +0 -352
  123. package/dist/memory/dreamer/review.js +0 -111
  124. package/dist/memory/durableAppend.js +0 -57
  125. package/dist/memory/factConflicts.js +0 -205
  126. package/dist/memory/factTerms.js +0 -321
  127. package/dist/memory/handoff.js +0 -363
  128. package/dist/memory/hiveNameHeader.cjs +0 -10
  129. package/dist/memory/inventory.js +0 -55
  130. package/dist/memory/invisibleText.js +0 -20
  131. package/dist/memory/lifecycle.js +0 -2014
  132. package/dist/memory/limits.js +0 -13
  133. package/dist/memory/requests.js +0 -183
  134. package/dist/memory/sourceName.js +0 -276
  135. package/dist/memory/statusFacts.js +0 -61
  136. package/dist/memory/whatsNew.js +0 -158
  137. package/dist/memory/writer.js +0 -176
  138. package/dist/memory/writerConflicts.js +0 -85
  139. package/dist/memorymap/memoryMap.js +0 -1594
  140. package/dist/memorymap/replacedJournal.js +0 -58
  141. package/dist/memorymap/replacements.js +0 -410
  142. package/dist/mvp/activity.js +0 -86
  143. package/dist/mvp/agentMemory.js +0 -1803
  144. package/dist/mvp/answerCheck.js +0 -211
  145. package/dist/mvp/auth.js +0 -531
  146. package/dist/mvp/cli.js +0 -86
  147. package/dist/mvp/client-store.js +0 -462
  148. package/dist/mvp/compileCache.js +0 -242
  149. package/dist/mvp/compileSeed.js +0 -33
  150. package/dist/mvp/config.js +0 -292
  151. package/dist/mvp/connectionStatus.js +0 -80
  152. package/dist/mvp/contextDistinct.js +0 -53
  153. package/dist/mvp/contextProfile.js +0 -87
  154. package/dist/mvp/contextRelevance.js +0 -413
  155. package/dist/mvp/directorySurface.js +0 -68
  156. package/dist/mvp/engine.js +0 -1078
  157. package/dist/mvp/exportAll.js +0 -393
  158. package/dist/mvp/factLanePack.js +0 -77
  159. package/dist/mvp/factMarkers.js +0 -65
  160. package/dist/mvp/factsLane.js +0 -203
  161. package/dist/mvp/hiddenSpans.js +0 -585
  162. package/dist/mvp/history.js +0 -414
  163. package/dist/mvp/hitDates.js +0 -31
  164. package/dist/mvp/hitFloor.js +0 -36
  165. package/dist/mvp/identity.js +0 -18
  166. package/dist/mvp/idle.js +0 -113
  167. package/dist/mvp/log.js +0 -118
  168. package/dist/mvp/memoryDir.js +0 -98
  169. package/dist/mvp/posthogMcp.js +0 -210
  170. package/dist/mvp/readScope.js +0 -104
  171. package/dist/mvp/readUsefulness.js +0 -222
  172. package/dist/mvp/sanitize.js +0 -48
  173. package/dist/mvp/schemaRejections.js +0 -91
  174. package/dist/mvp/scope.js +0 -127
  175. package/dist/mvp/server.js +0 -2712
  176. package/dist/mvp/serverCard.js +0 -38
  177. package/dist/mvp/storedSize.js +0 -139
  178. package/dist/mvp/telemetry.js +0 -474
  179. package/dist/net/poolFetch.js +0 -174
  180. package/dist/recall/actionRecall.js +0 -282
  181. package/dist/recall/autoRecall.js +0 -447
  182. package/dist/recall/searchRecall.js +0 -189
  183. package/dist/retrieval/adviceLines.js +0 -195
  184. package/dist/retrieval/adviceQuestion.js +0 -44
  185. package/dist/retrieval/conversationShape.js +0 -44
  186. package/dist/retrieval/dailyEntryTime.js +0 -55
  187. package/dist/retrieval/entityLookup.js +0 -162
  188. package/dist/retrieval/factsFirst.js +0 -188
  189. package/dist/retrieval/keyword.js +0 -84
  190. package/dist/retrieval/ledgerEventTime.js +0 -51
  191. package/dist/retrieval/ledgerLiveness.js +0 -83
  192. package/dist/retrieval/ledgerPresentation.js +0 -161
  193. package/dist/retrieval/noteIdentity.js +0 -57
  194. package/dist/retrieval/orchestrator.js +0 -1224
  195. package/dist/retrieval/plainWording.js +0 -236
  196. package/dist/retrieval/queryHelpers.js +0 -142
  197. package/dist/retrieval/questionSpan.js +0 -475
  198. package/dist/retrieval/rankingTrace.js +0 -184
  199. package/dist/retrieval/readerEvents.js +0 -690
  200. package/dist/retrieval/readerLayout.js +0 -422
  201. package/dist/retrieval/relativeTimeHints.js +0 -113
  202. package/dist/retrieval/semantic.js +0 -67
  203. package/dist/router/routerCache.js +0 -106
  204. package/dist/router/sessionRouter.js +0 -726
  205. package/dist/sharing/cli.js +0 -181
  206. package/dist/sharing/examples.js +0 -136
  207. package/dist/sharing/fields.js +0 -225
  208. package/dist/sharing/loop.js +0 -80
  209. package/dist/sharing/project.js +0 -327
  210. package/dist/sharing/send.js +0 -466
  211. package/dist/sharing/state.js +0 -222
  212. package/dist/sharing/wording.js +0 -48
  213. package/dist/toolRefusal.js +0 -120
  214. package/dist/watcher/watcher.js +0 -77
  215. /package/dist/{indexer/o200k.json → o200k.json} +0 -0
@@ -1,182 +0,0 @@
1
- "use strict";
2
- /**
3
- * Pure comparison policy between two eval runs. Extracted from the CLI
4
- * wrapper so the promotion logic itself lives in the frozen evaluator
5
- * trust domain (its hash is recorded in run manifests) and is unit tested.
6
- */
7
- Object.defineProperty(exports, "__esModule", { value: true });
8
- exports.QUALITY_METRICS = void 0;
9
- exports.compareRuns = compareRuns;
10
- exports.QUALITY_METRICS = [
11
- { key: 'hitAt1', higherBetter: true },
12
- { key: 'hitAt5', higherBetter: true },
13
- { key: 'recallAt5', higherBetter: true },
14
- { key: 'recallAllAt10', higherBetter: true },
15
- { key: 'mrr', higherBetter: true },
16
- { key: 'ndcgAt5', higherBetter: true },
17
- { key: 'contradictionFirstRate', higherBetter: false },
18
- { key: 'forbiddenRetrievalRate', higherBetter: false },
19
- { key: 'abstentionEmptyRate', higherBetter: true },
20
- ];
21
- const EFFICIENCY_METRICS = ['meanLatencyMs', 'p50LatencyMs', 'p95LatencyMs', 'meanContextTokens'];
22
- const MATERIAL_EFFICIENCY_METRICS = ['p95LatencyMs', 'meanContextTokens'];
23
- function same(a, b) {
24
- return JSON.stringify(a) === JSON.stringify(b);
25
- }
26
- function num(map, key) {
27
- const value = map[key];
28
- return typeof value === 'number' && Number.isFinite(value) ? value : null;
29
- }
30
- function delta(candidate, baseline) {
31
- return candidate === null || baseline === null ? null : Math.round((candidate - baseline) * 10000) / 10000;
32
- }
33
- /** Relative improvement of candidate over baseline; null when not computable. */
34
- function relativeGain(baseline, candidate) {
35
- if (baseline === null || candidate === null || baseline <= 0)
36
- return null;
37
- return (baseline - candidate) / baseline;
38
- }
39
- function compareRuns(baseline, candidate, options = {}) {
40
- const tolerance = options.maxQualityRegression ?? 0;
41
- const metricList = options.metrics ?? exports.QUALITY_METRICS;
42
- const minEfficiencyGain = options.minEfficiencyGain ?? 0.05;
43
- if (!Number.isFinite(tolerance) || tolerance < 0 || tolerance > 1) {
44
- throw new Error('maxQualityRegression must be between 0 and 1');
45
- }
46
- if (!Number.isFinite(minEfficiencyGain) || minEfficiencyGain < 0 || minEfficiencyGain > 1) {
47
- throw new Error('minEfficiencyGain must be between 0 and 1');
48
- }
49
- const invariantChecks = {
50
- benchmark: same(baseline.manifest.benchmark, candidate.manifest.benchmark),
51
- cases: same(baseline.manifest.casesSha256, candidate.manifest.casesSha256),
52
- fixture: same(baseline.manifest.fixtureSha256, candidate.manifest.fixtureSha256),
53
- topK: same(baseline.manifest.topK, candidate.manifest.topK),
54
- provider: same(baseline.manifest.provider, candidate.manifest.provider),
55
- chunking: same(baseline.manifest.chunking, candidate.manifest.chunking),
56
- evaluator: same(baseline.manifest.evaluatorSha256, candidate.manifest.evaluatorSha256),
57
- cleanTree: baseline.manifest.gitDirty === false && candidate.manifest.gitDirty === false,
58
- qualityEligible: baseline.manifest.qualityEligible === true && candidate.manifest.qualityEligible === true,
59
- };
60
- const reasons = [];
61
- const datasetCompatible = invariantChecks.benchmark && invariantChecks.cases && invariantChecks.fixture;
62
- if (!datasetCompatible) {
63
- throw new Error(`Cannot compare runs from different benchmark data: ${JSON.stringify(invariantChecks)}`);
64
- }
65
- if (!invariantChecks.evaluator) {
66
- if (!options.allowEvaluatorChange) {
67
- throw new Error('Evaluator changed between baseline and candidate (evaluatorSha256 mismatch). ' +
68
- 'The candidate cannot grade its own exam: re-run the candidate against the frozen evaluator, ' +
69
- 'or pass --allow-evaluator-change to accept a DESCRIPTIVE_ONLY comparison during evaluator development.');
70
- }
71
- reasons.push('Evaluator code changed between runs; measurement meaning differs.');
72
- }
73
- if (!invariantChecks.cleanTree) {
74
- if (!options.allowDirtyTree) {
75
- reasons.push('Working tree was dirty in at least one run; executed code does not match the recorded git SHA.');
76
- }
77
- else {
78
- reasons.push('Dirty working tree tolerated (--allow-dirty-tree).');
79
- }
80
- }
81
- if (!invariantChecks.qualityEligible) {
82
- reasons.push('At least one run used a non-quality provider (e.g. hash embeddings); infrastructure evidence only.');
83
- }
84
- const configCompatible = invariantChecks.topK && invariantChecks.provider && invariantChecks.chunking;
85
- if (!configCompatible) {
86
- reasons.push('Provider, topK, or chunking changed between runs.');
87
- }
88
- const regressionGate = options.regressionGate === true;
89
- const experimentCompatible = Boolean(invariantChecks.evaluator
90
- && (regressionGate || (invariantChecks.cleanTree && invariantChecks.qualityEligible))
91
- && (configCompatible || (!regressionGate && options.allowConfigChange)));
92
- const quality = Object.fromEntries(metricList.map(metric => {
93
- const baselineValue = num(baseline.summary.overall, metric.key);
94
- const candidateValue = num(candidate.summary.overall, metric.key);
95
- return [metric.key, { baseline: baselineValue, candidate: candidateValue, delta: delta(candidateValue, baselineValue) }];
96
- }));
97
- const efficiency = Object.fromEntries(EFFICIENCY_METRICS.map(key => {
98
- const baselineValue = num(baseline.summary.overall, key);
99
- const candidateValue = num(candidate.summary.overall, key);
100
- return [key, { baseline: baselineValue, candidate: candidateValue, delta: delta(candidateValue, baselineValue) }];
101
- }));
102
- let verdict;
103
- if (!experimentCompatible) {
104
- verdict = 'DESCRIPTIVE_ONLY';
105
- }
106
- else {
107
- const hasRegression = metricList.some(metric => {
108
- const { delta: d } = quality[metric.key];
109
- if (typeof d !== 'number')
110
- return false;
111
- return metric.higherBetter ? d < -tolerance : d > tolerance;
112
- });
113
- const hasQualityGain = metricList.some(metric => {
114
- const { delta: d } = quality[metric.key];
115
- if (typeof d !== 'number')
116
- return false;
117
- return metric.higherBetter ? d > tolerance : d < -tolerance;
118
- });
119
- const hasMaterialEfficiencyGain = MATERIAL_EFFICIENCY_METRICS.some(key => {
120
- const gain = relativeGain(efficiency[key].baseline, efficiency[key].candidate);
121
- return gain !== null && gain >= minEfficiencyGain;
122
- });
123
- if (hasRegression)
124
- verdict = 'REGRESSED';
125
- else if (hasQualityGain || hasMaterialEfficiencyGain)
126
- verdict = 'IMPROVED';
127
- else
128
- verdict = 'NEUTRAL';
129
- }
130
- // Category-collapse guard: an aggregate improvement must not hide one
131
- // category regressing beyond tolerance (independent-review finding).
132
- const categoryRegressions = [];
133
- const baselineCategories = baseline.summary.byCategory ?? {};
134
- const candidateCategories = candidate.summary.byCategory ?? {};
135
- // Both sides' categories: a category the baseline has and the candidate lost is a regression too.
136
- for (const name of new Set([...Object.keys(baselineCategories), ...Object.keys(candidateCategories)])) {
137
- const baselineMetrics = baselineCategories[name];
138
- const metrics = candidateCategories[name];
139
- if (!baselineMetrics)
140
- continue; // new category: nothing to regress against
141
- if (!metrics) {
142
- categoryRegressions.push(`${name}.missing`);
143
- continue;
144
- }
145
- for (const metric of metricList) {
146
- const b = num(baselineMetrics, metric.key);
147
- const c = num(metrics, metric.key);
148
- if (b === null || c === null)
149
- continue;
150
- const d = delta(c, b);
151
- if (typeof d === 'number' && (metric.higherBetter ? d < -tolerance : d > tolerance)) {
152
- categoryRegressions.push(`${name}.${metric.key}`);
153
- }
154
- }
155
- }
156
- for (const name of categoryRegressions) {
157
- reasons.push(`Category regression beyond tolerance: ${name}`);
158
- }
159
- if (categoryRegressions.length > 0) {
160
- // A category regression beyond tolerance is a rejection regardless of the
161
- // aggregate verdict: an overall gain must not hide a collapsing category.
162
- verdict = 'REGRESSED';
163
- }
164
- const note = regressionGate
165
- ? 'Regression gate against a committed baseline: this is a no-regression check, never a clean ablation or a promotion.'
166
- : experimentCompatible
167
- ? 'IMPROVED requires no quality regression beyond tolerance and a quality gain or a material (>= threshold) efficiency gain.'
168
- : 'Not a clean ablation: this comparison is descriptive only and cannot support promotion.';
169
- return {
170
- verdict,
171
- cleanAblation: experimentCompatible && !regressionGate,
172
- regressionGate,
173
- invariantChecks,
174
- tolerance,
175
- minEfficiencyGain,
176
- quality,
177
- efficiency,
178
- categoryRegressions,
179
- reasons,
180
- note,
181
- };
182
- }
@@ -1,145 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.canonicalSegment = exports.COMPILER_VERSION = exports.isHeaderOnly = void 0;
4
- exports.compileContext = compileContext;
5
- /**
6
- * Context700 campaign — MUTABLE context compiler.
7
- *
8
- * Campaign contract (docs/agents/autoresearch/context700/program.md):
9
- * - Target: <= 700 mean Renor memory-evidence tokens/query on LongMemEval-S full-500.
10
- * - Frozen quality floor: >= 428/500, no category regression > 2.0 pp vs N22 (430/500).
11
- * - ONLY this file is mutable after harness freeze.
12
- *
13
- * The compiler is gold-blind: it may use the question text/date, retrieval hits,
14
- * and structured extraction tables. It may NOT use gold answers, benchmark
15
- * question IDs (as decision keys), judge outputs, or correctness labels.
16
- *
17
- * v3 (structured-first + date-preserving compaction + recency raw fill):
18
- * Lesson from v1 (31/118 screen KILL): raw-first 55% budget + aggressive
19
- * compaction starved date-bearing stubs and chronicle breadth. v3:
20
- * 1. Query-gated structured rows FIRST (facts cap 12, chronicle cap 10, prefs cap 6).
21
- * 2. Raw excerpts: compact turn markers, PRESERVE Timestamp/date lines and
22
- * header stubs (they carry session dates), recency-ordered like N22.
23
- * 3. Fill remaining budget by rank; no minimum raw count (tables win ties).
24
- */
25
- const textTokens_1 = require("./textTokens");
26
- const canonicalSegment = (notePath) => notePath.replace(/^Sessions\//, '').replace(/-occ\d+\.md$/, '.md').replace(/\.md$/, '');
27
- exports.canonicalSegment = canonicalSegment;
28
- const isHeaderOnly = (text) => !text.includes('### Turn') && text.trim().length < 250;
29
- exports.isHeaderOnly = isHeaderOnly;
30
- exports.COMPILER_VERSION = 'v3-struct-first-date-keeping';
31
- const CONTENT_STOP = new Set('the a an of to for and or in on my i with is are was be that this it at as by from have has had do does did not no you your me we our they their he she his her about into over under then than so if but'.split(' '));
32
- const contentWords = (s) => {
33
- const out = new Set();
34
- for (const w of s.toLowerCase().matchAll(/[a-z0-9]{3,}/g)) {
35
- if (!CONTENT_STOP.has(w[0]))
36
- out.add(w[0]);
37
- }
38
- return out;
39
- };
40
- /** Compact date: "2023/05/30 (Tue) 23:40" -> "2023/05/30". */
41
- const compactDate = (d) => d ? d.replace(/\s*\((?:Mon|Tue|Wed|Thu|Fri|Sat|Sun)\).*/, '') : '';
42
- /**
43
- * v3 compaction: shorten turn markers to `[N-role]`, drop long session headers,
44
- * but PRESERVE Timestamp lines (dates are load-bearing for temporal/multi-session).
45
- */
46
- const compactText = (t) => t
47
- .replace(/^### Turn (\d+) — (User|Assistant)\s*\|?\s*/gim, '[$1-$2] ')
48
- .replace(/^Timestamp: (.*)$/gim, '@$1')
49
- .replace(/^# Session .*\n?/gim, '')
50
- .replace(/\n{3,}/g, '\n\n')
51
- .trim();
52
- const gateRows = (rows, qWords, cap) => {
53
- if (!rows || rows.length === 0)
54
- return [];
55
- const scored = [];
56
- for (const row of rows) {
57
- const text = Object.values(row).map(v => (typeof v === 'string' ? v : JSON.stringify(v) ?? '')).join(' ');
58
- let overlap = 0;
59
- for (const w of contentWords(text))
60
- if (qWords.has(w))
61
- overlap++;
62
- if (overlap > 0)
63
- scored.push({ score: overlap, row });
64
- }
65
- scored.sort((a, b) => b.score - a.score);
66
- return scored.slice(0, cap);
67
- };
68
- function compileContext(args) {
69
- const { question, hits, tables, dateMap, maxContextTokens } = args;
70
- const qWords = contentWords(question.question);
71
- const parts = { rawTokens: 0, factsTokens: 0, chronicleTokens: 0, prefsTokens: 0 };
72
- const excerpts = [];
73
- // --- Structured rows first (gated) ---
74
- let tokens = 0;
75
- let remaining = maxContextTokens;
76
- const factScored = gateRows(tables.facts ?? [], qWords, 12);
77
- if (factScored.length) {
78
- const lines = factScored.map(({ row: f }) => {
79
- const co = f.coexist ? ` (also: ${f.coexist})` : '';
80
- return `- ${f.value} ${f.unit ?? ''} ${f.object ?? ''}${f.event ? ` — ${f.event}` : ''}${co}`.replace(/\s+/g, ' ').trim();
81
- });
82
- const block = `Numeric facts:\n${lines.join('\n')}\nCite only values present here; otherwise use excerpts.`;
83
- const blockTokens = (0, textTokens_1.countTextTokens)(block);
84
- if (blockTokens <= remaining) {
85
- excerpts.push({ text: block, date: null });
86
- tokens += blockTokens;
87
- parts.factsTokens = blockTokens;
88
- remaining -= blockTokens;
89
- }
90
- }
91
- const chronScored = gateRows(tables.chronicle ?? [], qWords, 10);
92
- if (chronScored.length) {
93
- const lines = chronScored.map(({ row: e }) => `- ${e.when ?? ''}: ${e.event ?? ''}`.replace(/\s+/g, ' ').trim());
94
- const block = `Events:\n${lines.join('\n')}\nCite only events present here; otherwise use excerpts.`;
95
- const blockTokens = (0, textTokens_1.countTextTokens)(block);
96
- if (blockTokens <= remaining) {
97
- excerpts.push({ text: block, date: null });
98
- tokens += blockTokens;
99
- parts.chronicleTokens = blockTokens;
100
- remaining -= blockTokens;
101
- }
102
- }
103
- const prefScored = gateRows(tables.prefs ?? [], qWords, 6);
104
- if (prefScored.length) {
105
- const lines = prefScored.map(({ row: p }) => `- ${p.statement ?? ''}`.replace(/\s+/g, ' ').trim());
106
- const block = `Standing preferences:\n${lines.join('\n')}\nApply matching entries; otherwise use excerpts.`;
107
- const blockTokens = (0, textTokens_1.countTextTokens)(block);
108
- if (blockTokens <= remaining) {
109
- excerpts.push({ text: block, date: null });
110
- tokens += blockTokens;
111
- parts.prefsTokens = blockTokens;
112
- remaining -= blockTokens;
113
- }
114
- }
115
- const raws = [];
116
- const seen = new Set();
117
- for (const hit of args.hits) {
118
- const text = compactText((hit.text ?? '').trim());
119
- if (!text || seen.has(hit.text ?? ''))
120
- continue;
121
- const sessionDate = dateMap?.get(canonicalSegment(hit.notePath)) ?? null;
122
- seen.add(hit.text ?? '');
123
- raws.push({ text, date: sessionDate });
124
- }
125
- // N22 recency ordering (newest-first) before selection.
126
- if (args.recencyOrder) {
127
- raws.sort((a, b) => (b.date ?? '').localeCompare(a.date ?? ''));
128
- }
129
- for (const raw of raws) {
130
- const dateMark = raw.date ? `(${compactDate(raw.date)}) ` : '';
131
- const candidate = `${dateMark}${raw.text}`;
132
- const candidateTokens = (0, textTokens_1.countTextTokens)(candidate);
133
- if (tokens + candidateTokens > maxContextTokens)
134
- break;
135
- tokens += candidateTokens;
136
- excerpts.push({ text: candidate, date: raw.date });
137
- }
138
- parts.rawTokens = tokens - parts.factsTokens - parts.chronicleTokens - parts.prefsTokens;
139
- return {
140
- context: excerpts.map(excerpt => excerpt.text).join('\n\n'),
141
- tokens,
142
- parts,
143
- diagnostics: { compilerVersion: args.compilerVersion },
144
- };
145
- }