ziprin-context-optimizer 8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +48 -0
- package/bin/ziprin-context-mcp.js +34 -0
- package/bin/ziprin-context-setup.js +10 -0
- package/install-source.json +5 -0
- package/media/context-activity.svg +7 -0
- package/package.json +154 -0
- package/releases/ziprin-context-optimizer-8.0.0.vsix +0 -0
- package/scripts/install.mjs +204 -0
- package/scripts/package-vsix.mjs +38 -0
- package/scripts/paths.mjs +40 -0
- package/scripts/publish.mjs +37 -0
- package/scripts/update.mjs +101 -0
- package/src/adaptive-budget.js +46 -0
- package/src/analyzer.js +456 -0
- package/src/audit-history.js +137 -0
- package/src/bm25.js +132 -0
- package/src/collector.js +343 -0
- package/src/compression.js +67 -0
- package/src/config.js +28 -0
- package/src/context-memory.js +151 -0
- package/src/context-profiles.js +124 -0
- package/src/dependency-graph.js +90 -0
- package/src/estimate.js +109 -0
- package/src/eval-harness.js +76 -0
- package/src/extension.js +322 -0
- package/src/firewall.js +35 -0
- package/src/fts-index.js +319 -0
- package/src/gateway-api.js +404 -0
- package/src/git-recency.js +43 -0
- package/src/glob.js +42 -0
- package/src/identifier.js +37 -0
- package/src/inspector-panel.js +470 -0
- package/src/language.js +157 -0
- package/src/mcp-event-stream.js +179 -0
- package/src/mcp-health-cli.js +34 -0
- package/src/mcp-lifecycle-manager.js +427 -0
- package/src/mcp-server.js +372 -0
- package/src/mmr.js +57 -0
- package/src/pagerank.js +176 -0
- package/src/profiles.js +74 -0
- package/src/pruner.js +130 -0
- package/src/quality-guard.js +122 -0
- package/src/query-rewrite.js +32 -0
- package/src/relevance-scorer.js +197 -0
- package/src/repo-map.js +134 -0
- package/src/retrieve.js +350 -0
- package/src/serena.js +126 -0
- package/src/session-ledger.js +83 -0
- package/src/session-store.js +78 -0
- package/src/skeleton.js +129 -0
- package/src/slice-pack.js +70 -0
- package/src/supervisor.js +113 -0
- package/src/task-analyzer.js +189 -0
- package/src/tool-router.js +92 -0
- package/src/version.js +5 -0
- package/templates/ziprin-context-mcp.mjs +68 -0
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* Adaptive Token Budget Manager (v6 levels).
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const BUDGETS = {
|
|
8
|
+
simple_ui_fix: 5000,
|
|
9
|
+
backend_bug: 30000,
|
|
10
|
+
architecture_review: 100000,
|
|
11
|
+
mixed: 20000,
|
|
12
|
+
};
|
|
13
|
+
|
|
14
|
+
const LEVELS = {
|
|
15
|
+
simple_question: 3000,
|
|
16
|
+
bug_fix: 15000,
|
|
17
|
+
feature_development: 35000,
|
|
18
|
+
architecture_review: 100000,
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
function complexityToLevel(complexity, intent) {
|
|
22
|
+
if (complexity === 'architecture_review' || intent?.action === 'architecture') {
|
|
23
|
+
return 'architecture_review';
|
|
24
|
+
}
|
|
25
|
+
if (complexity === 'backend_bug' || intent?.action === 'fix') {
|
|
26
|
+
return 'bug_fix';
|
|
27
|
+
}
|
|
28
|
+
if (intent?.action === 'implement' || intent?.action === 'refactor' || intent?.action === 'migrate') {
|
|
29
|
+
return 'feature_development';
|
|
30
|
+
}
|
|
31
|
+
if (complexity === 'simple_ui_fix' || intent?.action === 'review') {
|
|
32
|
+
return 'simple_question';
|
|
33
|
+
}
|
|
34
|
+
return 'feature_development';
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function budgetForComplexity(complexity, profileCap, intent) {
|
|
38
|
+
const level = complexityToLevel(complexity, intent);
|
|
39
|
+
const levelBase = LEVELS[level] || BUDGETS[complexity] || BUDGETS.mixed;
|
|
40
|
+
const legacyBase = BUDGETS[complexity] || BUDGETS.mixed;
|
|
41
|
+
const base = Math.max(levelBase, legacyBase);
|
|
42
|
+
if (!profileCap) return base;
|
|
43
|
+
return Math.max(3000, Math.min(base, profileCap));
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
module.exports = { BUDGETS, LEVELS, budgetForComplexity, complexityToLevel };
|
package/src/analyzer.js
ADDED
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
const { tokens } = require('./estimate');
|
|
6
|
+
const { estimateContext } = require('./estimate');
|
|
7
|
+
const { getContextProfile, budgetFor } = require('./context-profiles');
|
|
8
|
+
const { collectWorkspace } = require('./collector');
|
|
9
|
+
const { analyzeTask } = require('./task-analyzer');
|
|
10
|
+
const { scoreCandidates, explainRemoval } = require('./relevance-scorer');
|
|
11
|
+
const { getOrBuildRepoMap, symbolPathsFromMap } = require('./repo-map');
|
|
12
|
+
const { buildDependencyBoost, protectDependencies } = require('./dependency-graph');
|
|
13
|
+
const { compressFile } = require('./compression');
|
|
14
|
+
const { budgetForComplexity } = require('./adaptive-budget');
|
|
15
|
+
const { qualityCheck, applyQualityActions } = require('./quality-guard');
|
|
16
|
+
const { loadMemory, effectivePathBoost, cooccurBoost } = require('./context-memory');
|
|
17
|
+
const { firewallDecision } = require('./firewall');
|
|
18
|
+
const { routeTask } = require('./tool-router');
|
|
19
|
+
const { retrieveCandidates } = require('./retrieve');
|
|
20
|
+
const { packSlice } = require('./slice-pack');
|
|
21
|
+
const { emitForTask } = require('./serena');
|
|
22
|
+
const { skeletonFile } = require('./skeleton');
|
|
23
|
+
const { loadLedger, splitFresh, remember } = require('./session-ledger');
|
|
24
|
+
const { VERSION } = require('./version');
|
|
25
|
+
|
|
26
|
+
const MAX_CONTEXT_FILES = 8;
|
|
27
|
+
|
|
28
|
+
function alwaysApplyTokens(root) {
|
|
29
|
+
try {
|
|
30
|
+
const est = estimateContext(root);
|
|
31
|
+
const agents = est.groups['cursor/agents']?.tokens || 0;
|
|
32
|
+
const skills = est.groups['cursor/skills']?.tokens || 0;
|
|
33
|
+
const rules = est.groups['cursor/rules']?.tokens || 0;
|
|
34
|
+
const catalogs = est.groups.catalogs?.tokens || 0;
|
|
35
|
+
return agents + skills + rules + catalogs;
|
|
36
|
+
} catch {
|
|
37
|
+
return 0;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function buildSnapshot(root, opts = {}) {
|
|
42
|
+
const dumpBodies = opts.dumpBodies === true;
|
|
43
|
+
const preferredProfile = opts.profileId || 'standard';
|
|
44
|
+
const prompt = String(opts.prompt || '').trim();
|
|
45
|
+
const task = String(opts.task || prompt || '').trim() || '(no prompt yet)';
|
|
46
|
+
const cpt = opts.charsPerToken || 4;
|
|
47
|
+
const sessionKey = opts.sessionId || opts.sessionKey || 'default';
|
|
48
|
+
const openFiles = opts.openFiles || [];
|
|
49
|
+
|
|
50
|
+
const memory = loadMemory(root);
|
|
51
|
+
const taskInfo = analyzeTask(task, preferredProfile, memory);
|
|
52
|
+
const profileId = opts.lockProfile ? preferredProfile : taskInfo.profileId;
|
|
53
|
+
const ctxProfile = getContextProfile(profileId);
|
|
54
|
+
const profileCap = budgetFor(profileId).maxOnDemandTokens;
|
|
55
|
+
const budget = budgetForComplexity(taskInfo.complexity, profileCap, taskInfo.intent);
|
|
56
|
+
|
|
57
|
+
const retrieved = retrieveCandidates(root, {
|
|
58
|
+
tokens: taskInfo.language?.originalTokens || taskInfo.tokens,
|
|
59
|
+
expanded: taskInfo.tokens,
|
|
60
|
+
symbols: taskInfo.symbols,
|
|
61
|
+
pathHints: taskInfo.pathHints,
|
|
62
|
+
domain: taskInfo.domain,
|
|
63
|
+
}, { openFiles });
|
|
64
|
+
|
|
65
|
+
const collected = collectWorkspace(root, profileId, cpt, {
|
|
66
|
+
domain: taskInfo.domain,
|
|
67
|
+
symbols: taskInfo.symbols || [],
|
|
68
|
+
pathHints: taskInfo.pathHints || [],
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
const byPath = new Map(collected.files.map((f) => [f.path, f]));
|
|
72
|
+
function ensureFile(rel) {
|
|
73
|
+
if (byPath.has(rel)) return;
|
|
74
|
+
const abs = path.join(root, rel);
|
|
75
|
+
if (!fs.existsSync(abs) || !fs.statSync(abs).isFile()) return;
|
|
76
|
+
const bytes = fs.statSync(abs).size;
|
|
77
|
+
const rec = { path: rel, bytes, tokens: tokens(bytes, cpt), seeded: true };
|
|
78
|
+
collected.files.unshift(rec);
|
|
79
|
+
byPath.set(rel, rec);
|
|
80
|
+
}
|
|
81
|
+
for (const item of retrieved.fused) ensureFile(item.path);
|
|
82
|
+
for (const p of retrieved.dirty) ensureFile(p);
|
|
83
|
+
for (const p of openFiles) ensureFile(p);
|
|
84
|
+
|
|
85
|
+
const repoMap = getOrBuildRepoMap(root, collected.symbolFiles || []);
|
|
86
|
+
for (const sym of taskInfo.symbols || []) {
|
|
87
|
+
for (const p of symbolPathsFromMap(repoMap, sym)) ensureFile(p);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
const route = routeTask(task);
|
|
91
|
+
const pathBoost = effectivePathBoost(memory);
|
|
92
|
+
|
|
93
|
+
const firewalled = [];
|
|
94
|
+
const survivors = [];
|
|
95
|
+
for (const f of collected.files) {
|
|
96
|
+
const d = firewallDecision(f.path, f.bytes);
|
|
97
|
+
if (d.action === 'block') {
|
|
98
|
+
firewalled.push({ file: f.path, reason: d.reason, tokenSaving: f.tokens, action: 'block' });
|
|
99
|
+
} else {
|
|
100
|
+
survivors.push({ ...f, firewall: d });
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
for (const ig of collected.ignored) {
|
|
104
|
+
firewalled.push({ ...ig, action: 'block' });
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
const tokensBefore =
|
|
108
|
+
survivors.reduce((n, f) => n + f.tokens, 0) +
|
|
109
|
+
firewalled.reduce((n, f) => n + (f.tokenSaving || 0), 0);
|
|
110
|
+
|
|
111
|
+
const scorerOpts = {
|
|
112
|
+
taskTokens: taskInfo.tokens,
|
|
113
|
+
category: taskInfo.category,
|
|
114
|
+
memoryBoost: pathBoost,
|
|
115
|
+
domain: taskInfo.domain,
|
|
116
|
+
symbols: taskInfo.symbols || [],
|
|
117
|
+
pathHints: taskInfo.pathHints || [],
|
|
118
|
+
action: taskInfo.intent?.action,
|
|
119
|
+
prompt: task,
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
let scored = scoreCandidates(survivors, scorerOpts);
|
|
123
|
+
if (memory.negativeBoost) {
|
|
124
|
+
scored = scored.map((s) => {
|
|
125
|
+
const n = memory.negativeBoost[s.path] || 0;
|
|
126
|
+
if (!n) return s;
|
|
127
|
+
return { ...s, final_score: s.final_score - n, keep: s.final_score - n >= 15 };
|
|
128
|
+
}).sort((a, b) => b.final_score - a.final_score);
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const topSeeds = scored.filter((s) => s.final_score > 20).slice(0, 25).map((s) => s.path);
|
|
132
|
+
const { boost, edges } = buildDependencyBoost(root, topSeeds, 40);
|
|
133
|
+
scored = scoreCandidates(survivors, { ...scorerOpts, dependencyBoost: boost });
|
|
134
|
+
|
|
135
|
+
const rrfBoost = new Map(retrieved.fused.map((x, i) => [x.path, 40 - i]));
|
|
136
|
+
scored = scored.map((s) => {
|
|
137
|
+
const extra = rrfBoost.get(s.path) || 0;
|
|
138
|
+
const dirty = retrieved.dirty.includes(s.path) ? 25 : 0;
|
|
139
|
+
const open = openFiles.includes(s.path) ? 30 : 0;
|
|
140
|
+
const co = cooccurBoost(memory, s.path, topSeeds.slice(0, 6));
|
|
141
|
+
const final_score = s.final_score + extra + dirty + open + co;
|
|
142
|
+
return { ...s, final_score, keep: final_score >= 15 };
|
|
143
|
+
}).sort((a, b) => b.final_score - a.final_score);
|
|
144
|
+
|
|
145
|
+
const slice = packSlice(
|
|
146
|
+
scored.filter((s) => s.keep).map((s) => s.path),
|
|
147
|
+
survivors.map((f) => f.path),
|
|
148
|
+
{ maxFiles: MAX_CONTEXT_FILES }
|
|
149
|
+
);
|
|
150
|
+
const sliceSet = new Set(slice);
|
|
151
|
+
|
|
152
|
+
const selected = [];
|
|
153
|
+
const removedItems = [...firewalled];
|
|
154
|
+
let used = 0;
|
|
155
|
+
const prefer = [
|
|
156
|
+
...slice,
|
|
157
|
+
...scored.map((s) => s.path).filter((p) => !sliceSet.has(p)),
|
|
158
|
+
];
|
|
159
|
+
const seenSel = new Set();
|
|
160
|
+
for (const p of prefer) {
|
|
161
|
+
if (seenSel.has(p)) continue;
|
|
162
|
+
const s = scored.find((x) => x.path === p);
|
|
163
|
+
const file = survivors.find((f) => f.path === p);
|
|
164
|
+
if (!file) continue;
|
|
165
|
+
seenSel.add(p);
|
|
166
|
+
const fw = file.firewall || firewallDecision(file.path, file.bytes);
|
|
167
|
+
const cost = fw.action === 'summarize' ? Math.min(file.tokens, 500) : file.tokens;
|
|
168
|
+
const inSlice = sliceSet.has(p);
|
|
169
|
+
if (!inSlice && s && !s.keep && s.final_score < 15) {
|
|
170
|
+
removedItems.push({
|
|
171
|
+
file: s.path,
|
|
172
|
+
reason: explainRemoval(s.breakdown, s.final_score, `low relevance (score ${s.final_score})`),
|
|
173
|
+
tokenSaving: file.tokens,
|
|
174
|
+
action: 'prune',
|
|
175
|
+
score: s.final_score,
|
|
176
|
+
});
|
|
177
|
+
continue;
|
|
178
|
+
}
|
|
179
|
+
if (used + cost > budget && selected.length >= 3 && !inSlice) {
|
|
180
|
+
removedItems.push({
|
|
181
|
+
file: file.path,
|
|
182
|
+
reason: 'exceeded adaptive token budget',
|
|
183
|
+
tokenSaving: file.tokens,
|
|
184
|
+
action: 'budget',
|
|
185
|
+
score: s?.final_score,
|
|
186
|
+
});
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
if (selected.length >= MAX_CONTEXT_FILES && !inSlice) {
|
|
190
|
+
removedItems.push({
|
|
191
|
+
file: file.path,
|
|
192
|
+
reason: 'cap: max context files',
|
|
193
|
+
tokenSaving: file.tokens,
|
|
194
|
+
action: 'cap',
|
|
195
|
+
score: s?.final_score,
|
|
196
|
+
});
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
selected.push({ path: file.path, tokens: cost, score: s?.final_score || 0, compress: fw.action === 'summarize' });
|
|
200
|
+
used += cost;
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const protectedSet = protectDependencies(
|
|
204
|
+
selected.map((s) => s.path),
|
|
205
|
+
boost,
|
|
206
|
+
scored
|
|
207
|
+
);
|
|
208
|
+
let selectedPaths = protectedSet.selected;
|
|
209
|
+
|
|
210
|
+
const quality = qualityCheck({
|
|
211
|
+
taskTokens: taskInfo.tokens,
|
|
212
|
+
selected: selectedPaths,
|
|
213
|
+
restored: protectedSet.restored,
|
|
214
|
+
scores: scored,
|
|
215
|
+
budget,
|
|
216
|
+
usedTokens: used,
|
|
217
|
+
symbols: taskInfo.symbols || [],
|
|
218
|
+
pathHints: taskInfo.pathHints || [],
|
|
219
|
+
domain: taskInfo.domain,
|
|
220
|
+
});
|
|
221
|
+
selectedPaths = applyQualityActions(selectedPaths, scored, quality).slice(0, MAX_CONTEXT_FILES + 2);
|
|
222
|
+
|
|
223
|
+
if (!quality.ok && taskInfo.domain?.priorityPaths?.length) {
|
|
224
|
+
const domainHit = selectedPaths.some((p) =>
|
|
225
|
+
taskInfo.domain.priorityPaths.some((dp) => p.toLowerCase().includes(dp.toLowerCase())),
|
|
226
|
+
);
|
|
227
|
+
if (!domainHit) {
|
|
228
|
+
for (const p of collected.symbolFiles || []) {
|
|
229
|
+
if (!selectedPaths.includes(p)) selectedPaths.unshift(p);
|
|
230
|
+
}
|
|
231
|
+
selectedPaths = selectedPaths.slice(0, MAX_CONTEXT_FILES + 2);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
for (const hint of taskInfo.pathHints || []) {
|
|
236
|
+
const hintHit = selectedPaths.some((p) => p.toLowerCase().includes(hint));
|
|
237
|
+
if (!hintHit) {
|
|
238
|
+
for (const f of collected.files) {
|
|
239
|
+
if (f.path.toLowerCase().includes(hint) && !selectedPaths.includes(f.path)) {
|
|
240
|
+
selectedPaths.unshift(f.path);
|
|
241
|
+
break;
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
selectedPaths = [...new Set(selectedPaths)].slice(0, MAX_CONTEXT_FILES);
|
|
247
|
+
|
|
248
|
+
const ledger = loadLedger(root);
|
|
249
|
+
let readNow = selectedPaths.slice();
|
|
250
|
+
let alreadyInContext = [];
|
|
251
|
+
if (opts.sessionId || opts.sessionKey) {
|
|
252
|
+
const split = splitFresh(selectedPaths, ledger, sessionKey);
|
|
253
|
+
readNow = split.fresh;
|
|
254
|
+
alreadyInContext = split.alreadyInContext;
|
|
255
|
+
remember(root, sessionKey, selectedPaths);
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
const vsAlways = alwaysApplyTokens(root);
|
|
259
|
+
const tokensAfterPlan = selectedPaths.reduce((n, p) => {
|
|
260
|
+
const f = survivors.find((x) => x.path === p);
|
|
261
|
+
return n + (f?.tokens || 0);
|
|
262
|
+
}, 0);
|
|
263
|
+
|
|
264
|
+
const serenaEmit = emitForTask(taskInfo.symbols || [], readNow);
|
|
265
|
+
const serenaStrings = (taskInfo.symbols || []).slice(0, 8).map((s) => `find_symbol:${s}`);
|
|
266
|
+
|
|
267
|
+
const decisionReport = {
|
|
268
|
+
intent: taskInfo.intent,
|
|
269
|
+
domain: taskInfo.domain,
|
|
270
|
+
seedDirs: collected.seedDirs || [],
|
|
271
|
+
symbolFiles: collected.symbolFiles || [],
|
|
272
|
+
retrieve: {
|
|
273
|
+
dirty: retrieved.dirty.slice(0, 12),
|
|
274
|
+
fusedTop: retrieved.fused.slice(0, 12).map((x) => x.path),
|
|
275
|
+
ftsHits: (retrieved.ftsHits || []).slice(0, 8),
|
|
276
|
+
cascadeSkip: Boolean(retrieved.cascadeSkip),
|
|
277
|
+
stages: retrieved.stages || {},
|
|
278
|
+
ftsMeta: retrieved.ftsMeta || null,
|
|
279
|
+
},
|
|
280
|
+
slice,
|
|
281
|
+
topSelected: selectedPaths.slice(0, 8).map((p) => {
|
|
282
|
+
const s = scored.find((x) => x.path === p);
|
|
283
|
+
return { path: p, score: s?.final_score, why: s?.whySelected, confidence: s?.confidence };
|
|
284
|
+
}),
|
|
285
|
+
topRemoved: removedItems.slice(0, 8).map((r) => ({
|
|
286
|
+
file: r.file,
|
|
287
|
+
reason: r.reason,
|
|
288
|
+
score: r.score,
|
|
289
|
+
})),
|
|
290
|
+
quality: { ok: quality.ok, issues: quality.issues },
|
|
291
|
+
};
|
|
292
|
+
|
|
293
|
+
const selectedMeta = [];
|
|
294
|
+
const contextParts = [
|
|
295
|
+
`# Ziprin Context MCP ${VERSION}`,
|
|
296
|
+
`task: ${task}`,
|
|
297
|
+
`classification: ${taskInfo.label} (${taskInfo.category})`,
|
|
298
|
+
`profile: ${ctxProfile.id}`,
|
|
299
|
+
`budget: ${budget} (${taskInfo.complexity})`,
|
|
300
|
+
`mode: ${dumpBodies ? 'dump' : 'map'}`,
|
|
301
|
+
`prompt:`,
|
|
302
|
+
prompt || '(empty)',
|
|
303
|
+
'',
|
|
304
|
+
`# Read now`,
|
|
305
|
+
...readNow.map((p) => `- ${p}`),
|
|
306
|
+
'',
|
|
307
|
+
`# Already in context`,
|
|
308
|
+
...(alreadyInContext.length ? alreadyInContext.map((p) => `- ${p}`) : ['- (none)']),
|
|
309
|
+
'',
|
|
310
|
+
`# Serena first`,
|
|
311
|
+
...serenaEmit.slice(0, 8).map((c) => `- ${c.tool} ${c.name_path_pattern || c.relative_path || ''}`),
|
|
312
|
+
'',
|
|
313
|
+
`# Do not read`,
|
|
314
|
+
'- .cursor/agents/**',
|
|
315
|
+
'- .cursor/skills/**',
|
|
316
|
+
'- tooling/project-index.json',
|
|
317
|
+
];
|
|
318
|
+
|
|
319
|
+
let tokensAfter = 0;
|
|
320
|
+
if (dumpBodies) {
|
|
321
|
+
for (const p of selectedPaths) {
|
|
322
|
+
const file = survivors.find((f) => f.path === p) || { path: p, tokens: 0, bytes: 0 };
|
|
323
|
+
const shouldCompress =
|
|
324
|
+
(file.firewall && file.firewall.action === 'summarize') || (file.bytes || 0) > 40_000;
|
|
325
|
+
const piece = compressFile(root, p, shouldCompress ? 1800 : 3500);
|
|
326
|
+
selectedMeta.push({
|
|
327
|
+
path: p,
|
|
328
|
+
mode: piece.mode,
|
|
329
|
+
tokens: piece.tokenEstimate,
|
|
330
|
+
score: scored.find((s) => s.path === p)?.final_score,
|
|
331
|
+
signatures: (piece.signatures || []).map((s) => s.name),
|
|
332
|
+
exports: piece.exports || [],
|
|
333
|
+
});
|
|
334
|
+
tokensAfter += piece.tokenEstimate;
|
|
335
|
+
contextParts.push(`\n----- ${p} (${piece.mode}) -----\n`);
|
|
336
|
+
contextParts.push(piece.text);
|
|
337
|
+
}
|
|
338
|
+
} else {
|
|
339
|
+
for (const p of selectedPaths) {
|
|
340
|
+
const file = survivors.find((f) => f.path === p) || { path: p, tokens: 80, bytes: 0 };
|
|
341
|
+
const skel = skeletonFile(root, p);
|
|
342
|
+
const est = Math.min(120, file.tokens || skel.tokenEstimate || 80);
|
|
343
|
+
selectedMeta.push({
|
|
344
|
+
path: p,
|
|
345
|
+
mode: 'map',
|
|
346
|
+
tokens: est,
|
|
347
|
+
score: scored.find((s) => s.path === p)?.final_score,
|
|
348
|
+
signatures: (skel.signatures || []).map((s) => s.name),
|
|
349
|
+
exports: (skel.signatures || []).map((s) => s.name),
|
|
350
|
+
});
|
|
351
|
+
tokensAfter += est;
|
|
352
|
+
const sig = (skel.signatures || []).slice(0, 8).map((s) => s.name).join(', ');
|
|
353
|
+
contextParts.push(`- ${p} :: ${sig || skel.mode}`);
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
const reductionPct =
|
|
358
|
+
tokensBefore > 0 ? Math.round((1 - tokensAfter / tokensBefore) * 100) : 0;
|
|
359
|
+
const vsAlwaysPct = vsAlways > 0 ? Math.round((1 - tokensAfter / (tokensAfter + vsAlways)) * 100) : reductionPct;
|
|
360
|
+
|
|
361
|
+
const analysis = [
|
|
362
|
+
`Task: ${taskInfo.label} / complexity=${taskInfo.complexity}`,
|
|
363
|
+
`Tokens collected ${tokensBefore} → map ${tokensAfter} (−${reductionPct}%). vsAlwaysApply extra ${vsAlways}.`,
|
|
364
|
+
`Selected ${selectedPaths.length}; slice ${slice.length}; dirty ${retrieved.dirty.length}; cascade=${retrieved.cascadeSkip}.`,
|
|
365
|
+
quality.qualityNote,
|
|
366
|
+
route.recommendation?.why ? `Router: ${route.recommendation.why}` : '',
|
|
367
|
+
'Return paths + signatures. Call Serena find_symbol before read_file. Never @folder.',
|
|
368
|
+
]
|
|
369
|
+
.filter(Boolean)
|
|
370
|
+
.join('\n');
|
|
371
|
+
|
|
372
|
+
return {
|
|
373
|
+
id: `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
|
374
|
+
timestamp: new Date().toISOString(),
|
|
375
|
+
version: VERSION,
|
|
376
|
+
task,
|
|
377
|
+
prompt: prompt || task,
|
|
378
|
+
taskClassification: taskInfo.category,
|
|
379
|
+
taskLabel: taskInfo.label,
|
|
380
|
+
intent: taskInfo.intent,
|
|
381
|
+
domainId: taskInfo.domain?.id || null,
|
|
382
|
+
profileId: ctxProfile.id,
|
|
383
|
+
filesIncluded: selectedPaths,
|
|
384
|
+
filesExcluded: removedItems.map((r) => r.file),
|
|
385
|
+
readNow,
|
|
386
|
+
alreadyInContext,
|
|
387
|
+
serena: serenaStrings,
|
|
388
|
+
serenaEmit,
|
|
389
|
+
doNotRead: ['.cursor/agents/**', '.cursor/skills/**', 'tooling/project-index.json', '**/reports/**'],
|
|
390
|
+
rulesApplied: collected.rules.map((r) => r.path + (r.alwaysApply ? ' (alwaysApply)' : '')),
|
|
391
|
+
tokensBeforePrune: tokensBefore,
|
|
392
|
+
tokensAfterPrune: tokensAfter,
|
|
393
|
+
removedItems,
|
|
394
|
+
finalContext: contextParts.join('\n'),
|
|
395
|
+
result: analysis,
|
|
396
|
+
qualityReport: quality,
|
|
397
|
+
decisionReport,
|
|
398
|
+
tokenAnalysis: {
|
|
399
|
+
before: tokensBefore,
|
|
400
|
+
after: tokensAfter,
|
|
401
|
+
budget,
|
|
402
|
+
reductionPct,
|
|
403
|
+
vsNaiveCollect: reductionPct,
|
|
404
|
+
vsAlwaysApplyTokens: vsAlways,
|
|
405
|
+
vsAlwaysApplyPct: vsAlwaysPct,
|
|
406
|
+
vsSelected: tokensAfterPlan,
|
|
407
|
+
complexity: taskInfo.complexity,
|
|
408
|
+
level: taskInfo.intent?.action,
|
|
409
|
+
dumpBodies,
|
|
410
|
+
stages: retrieved.stages || {},
|
|
411
|
+
},
|
|
412
|
+
selectedMeta,
|
|
413
|
+
dependencyEdges: edges.slice(0, 80),
|
|
414
|
+
live: {
|
|
415
|
+
truncated: collected.truncated,
|
|
416
|
+
route,
|
|
417
|
+
rules: collected.rules,
|
|
418
|
+
budget,
|
|
419
|
+
taskInfo,
|
|
420
|
+
scoredTop: scored.slice(0, 20),
|
|
421
|
+
retrieved,
|
|
422
|
+
},
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
|
|
426
|
+
function toSession(snapshot) {
|
|
427
|
+
return {
|
|
428
|
+
id: snapshot.id,
|
|
429
|
+
timestamp: snapshot.timestamp,
|
|
430
|
+
task: snapshot.task,
|
|
431
|
+
prompt: snapshot.prompt,
|
|
432
|
+
taskClassification: snapshot.taskClassification,
|
|
433
|
+
taskLabel: snapshot.taskLabel,
|
|
434
|
+
intent: snapshot.intent,
|
|
435
|
+
domainId: snapshot.domainId,
|
|
436
|
+
filesIncluded: snapshot.filesIncluded,
|
|
437
|
+
filesExcluded: snapshot.filesExcluded,
|
|
438
|
+
readNow: snapshot.readNow,
|
|
439
|
+
alreadyInContext: snapshot.alreadyInContext,
|
|
440
|
+
serena: snapshot.serena,
|
|
441
|
+
serenaEmit: snapshot.serenaEmit,
|
|
442
|
+
doNotRead: snapshot.doNotRead,
|
|
443
|
+
rulesApplied: snapshot.rulesApplied,
|
|
444
|
+
tokensBeforePrune: snapshot.tokensBeforePrune,
|
|
445
|
+
tokensAfterPrune: snapshot.tokensAfterPrune,
|
|
446
|
+
removedItems: snapshot.removedItems,
|
|
447
|
+
finalContext: snapshot.finalContext,
|
|
448
|
+
result: snapshot.result,
|
|
449
|
+
qualityReport: snapshot.qualityReport,
|
|
450
|
+
decisionReport: snapshot.decisionReport,
|
|
451
|
+
tokenAnalysis: snapshot.tokenAnalysis,
|
|
452
|
+
profileId: snapshot.profileId,
|
|
453
|
+
};
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
module.exports = { buildSnapshot, toSession, VERSION };
|
|
@@ -0,0 +1,137 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
const fs = require('fs');
|
|
4
|
+
const path = require('path');
|
|
5
|
+
|
|
6
|
+
const AUDIT_REL = '.cursor/context-optimizer-audit.jsonl';
|
|
7
|
+
|
|
8
|
+
function auditFilePath(root) {
|
|
9
|
+
return path.join(root, AUDIT_REL);
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function readEntries(root) {
|
|
13
|
+
const file = auditFilePath(root);
|
|
14
|
+
if (!fs.existsSync(file)) return [];
|
|
15
|
+
const lines = fs.readFileSync(file, 'utf8').split(/\r?\n/).filter(Boolean);
|
|
16
|
+
const out = [];
|
|
17
|
+
for (const line of lines) {
|
|
18
|
+
try {
|
|
19
|
+
out.push(JSON.parse(line));
|
|
20
|
+
} catch {
|
|
21
|
+
/* skip corrupt line */
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return out;
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function writeEntries(root, entries) {
|
|
28
|
+
const file = auditFilePath(root);
|
|
29
|
+
fs.mkdirSync(path.dirname(file), { recursive: true });
|
|
30
|
+
const text = entries.map((e) => JSON.stringify(e)).join('\n') + (entries.length ? '\n' : '');
|
|
31
|
+
fs.writeFileSync(file, text, 'utf8');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function makeEntry(kind, payload) {
|
|
35
|
+
return {
|
|
36
|
+
id: `${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
|
|
37
|
+
at: new Date().toISOString(),
|
|
38
|
+
kind,
|
|
39
|
+
...payload,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
function appendEntry(root, entry, maxEntries = 100) {
|
|
44
|
+
if (!root) return entry;
|
|
45
|
+
const entries = readEntries(root);
|
|
46
|
+
entries.push(entry);
|
|
47
|
+
const trimmed = entries.slice(-Math.max(1, maxEntries));
|
|
48
|
+
writeEntries(root, trimmed);
|
|
49
|
+
return entry;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function clearEntries(root) {
|
|
53
|
+
const file = auditFilePath(root);
|
|
54
|
+
if (fs.existsSync(file)) fs.unlinkSync(file);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
function listRecent(root, limit = 15) {
|
|
58
|
+
return readEntries(root).slice(-limit).reverse();
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function formatEntryLabel(entry) {
|
|
62
|
+
const t = entry.at ? entry.at.replace('T', ' ').slice(0, 19) : '?';
|
|
63
|
+
const task = entry.task ? ` — ${String(entry.task).slice(0, 40)}` : '';
|
|
64
|
+
return `[${t}] ${entry.kind}${task}`;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
function buildPlanAudit(task, profileId, route, pruned, estimate) {
|
|
68
|
+
const skipSteps = (pruned.steps || []).filter((s) => s.action === 'skip');
|
|
69
|
+
const skipped = skipSteps.map((s) => s.what).filter(Boolean);
|
|
70
|
+
if (route.neverDump?.length) {
|
|
71
|
+
for (const s of route.neverDump) if (!skipped.includes(s)) skipped.push(s);
|
|
72
|
+
}
|
|
73
|
+
return makeEntry('planChat', {
|
|
74
|
+
task,
|
|
75
|
+
profile: profileId,
|
|
76
|
+
summary: {
|
|
77
|
+
totalTokens: estimate?.total?.tokens ?? null,
|
|
78
|
+
alwaysApplyCount: estimate?.alwaysApplyTrue?.length ?? null,
|
|
79
|
+
recommendedTool: route.recommendation?.tool ?? null,
|
|
80
|
+
skipped,
|
|
81
|
+
selectedPaths: (pruned.pruned?.selected || []).map((s) => s.path),
|
|
82
|
+
usedTokens: pruned.pruned?.usedTokens ?? null,
|
|
83
|
+
budget: pruned.pruned?.budget ?? null,
|
|
84
|
+
},
|
|
85
|
+
detail: { route, pruned },
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
function buildEstimateAudit(profileId, report) {
|
|
90
|
+
return makeEntry('estimate', {
|
|
91
|
+
profile: profileId,
|
|
92
|
+
summary: {
|
|
93
|
+
totalTokens: report.total.tokens,
|
|
94
|
+
alwaysApplyCount: report.alwaysApplyTrue.length,
|
|
95
|
+
groups: report.groups,
|
|
96
|
+
},
|
|
97
|
+
detail: { alwaysApplyTrue: report.alwaysApplyTrue },
|
|
98
|
+
});
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function buildRefreshAudit(liveState) {
|
|
102
|
+
return makeEntry('refresh', {
|
|
103
|
+
task: liveState?.task,
|
|
104
|
+
profile: liveState?.profile?.id,
|
|
105
|
+
summary: {
|
|
106
|
+
totalTokens: liveState?.estimate?.total?.tokens ?? null,
|
|
107
|
+
alwaysApplyCount: liveState?.estimate?.alwaysApplyTrue?.length ?? null,
|
|
108
|
+
recommendedTool: liveState?.route?.recommendation?.tool ?? null,
|
|
109
|
+
usedTokens: liveState?.pruned?.pruned?.usedTokens ?? null,
|
|
110
|
+
budget: liveState?.pruned?.pruned?.budget ?? null,
|
|
111
|
+
},
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
function buildInstallAudit(result) {
|
|
116
|
+
return makeEntry('installTokenBudget', {
|
|
117
|
+
summary: {
|
|
118
|
+
stripped: result.stripped?.length ?? 0,
|
|
119
|
+
flipped: result.flipped?.length ?? 0,
|
|
120
|
+
},
|
|
121
|
+
detail: result,
|
|
122
|
+
});
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
module.exports = {
|
|
126
|
+
AUDIT_REL,
|
|
127
|
+
auditFilePath,
|
|
128
|
+
readEntries,
|
|
129
|
+
appendEntry,
|
|
130
|
+
clearEntries,
|
|
131
|
+
listRecent,
|
|
132
|
+
formatEntryLabel,
|
|
133
|
+
buildPlanAudit,
|
|
134
|
+
buildEstimateAudit,
|
|
135
|
+
buildRefreshAudit,
|
|
136
|
+
buildInstallAudit,
|
|
137
|
+
};
|