scalix 0.0.1 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +29 -0
- package/CHANGELOG.md +122 -0
- package/LICENSE +100 -0
- package/README.md +300 -2
- package/bin/scalix.js +19 -0
- package/dist/agent-definitions.js +100 -0
- package/dist/agent-registry.js +226 -0
- package/dist/agent-replay.js +166 -0
- package/dist/agent.js +1130 -0
- package/dist/api-client.js +77 -0
- package/dist/auto-verify.js +136 -0
- package/dist/change-tracker.js +53 -0
- package/dist/cli.js +2454 -0
- package/dist/commands.js +71 -0
- package/dist/config.js +16 -0
- package/dist/constants.js +358 -0
- package/dist/context-builder.js +158 -0
- package/dist/context-manager.js +148 -0
- package/dist/context-retriever.js +257 -0
- package/dist/cost-tracker.js +85 -0
- package/dist/diff.js +140 -0
- package/dist/git-context.js +183 -0
- package/dist/git-undo.js +67 -0
- package/dist/grounding-checker.js +154 -0
- package/dist/headless.js +743 -0
- package/dist/hooks.js +100 -0
- package/dist/injection-detector.js +86 -0
- package/dist/ipc-server.js +232 -0
- package/dist/logger.js +45 -0
- package/dist/markdown.js +41 -0
- package/dist/mcp.js +49 -0
- package/dist/model-router.js +68 -0
- package/dist/plugins.js +132 -0
- package/dist/project-index.js +376 -0
- package/dist/project-memory.js +165 -0
- package/dist/safe-math.js +194 -0
- package/dist/sandbox.js +403 -0
- package/dist/scoped-rules.js +82 -0
- package/dist/sessions.js +106 -0
- package/dist/settings.js +260 -0
- package/dist/sub-agent.js +154 -0
- package/dist/task-board.js +105 -0
- package/dist/token-counter.js +47 -0
- package/dist/tool-helper.js +97 -0
- package/dist/tool-selector.js +77 -0
- package/dist/tools-advanced.js +487 -0
- package/dist/tools-browser.js +534 -0
- package/dist/tools-extra.js +390 -0
- package/dist/tools-vision.js +116 -0
- package/dist/tools-voice.js +147 -0
- package/dist/tools-web.js +346 -0
- package/dist/tools.js +380 -0
- package/dist/types.js +1 -0
- package/dist/update-checker.js +111 -0
- package/dist/worktree.js +172 -0
- package/docs/DISTRIBUTION.md +301 -0
- package/package.json +71 -6
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
import { messageContentToString } from './agent.js';
|
|
2
|
+
import { estimateTokens, estimateMessageTokens as _estimateMessageTokens } from './token-counter.js';
|
|
3
|
+
import { CONTEXT_DEFAULTS, COMPRESSION_DEFAULTS } from './constants.js';
|
|
4
|
+
export { estimateTokens };
|
|
5
|
+
export function estimateMessageTokens(messages) {
|
|
6
|
+
return _estimateMessageTokens(messages);
|
|
7
|
+
}
|
|
8
|
+
export class ContextManager {
|
|
9
|
+
maxContextTokens;
|
|
10
|
+
compressionThreshold;
|
|
11
|
+
minRecentMessages;
|
|
12
|
+
maxToolResultTokens;
|
|
13
|
+
charsPerToken;
|
|
14
|
+
slidingWindowTarget;
|
|
15
|
+
constructor(config) {
|
|
16
|
+
this.maxContextTokens = config?.maxContextTokens ?? CONTEXT_DEFAULTS.maxContextTokens;
|
|
17
|
+
this.compressionThreshold = config?.compressionThreshold ?? CONTEXT_DEFAULTS.compressionThreshold;
|
|
18
|
+
this.minRecentMessages = config?.minRecentMessages ?? CONTEXT_DEFAULTS.minRecentMessages;
|
|
19
|
+
this.maxToolResultTokens = config?.maxToolResultTokens ?? CONTEXT_DEFAULTS.maxToolResultTokens;
|
|
20
|
+
this.charsPerToken = config?.charsPerToken ?? CONTEXT_DEFAULTS.charsPerToken;
|
|
21
|
+
this.slidingWindowTarget = config?.slidingWindowTarget ?? CONTEXT_DEFAULTS.slidingWindowTarget;
|
|
22
|
+
}
|
|
23
|
+
setMaxContextTokens(tokens) {
|
|
24
|
+
if (tokens > 0)
|
|
25
|
+
this.maxContextTokens = tokens;
|
|
26
|
+
}
|
|
27
|
+
shouldCompress(messages) {
|
|
28
|
+
const tokens = estimateMessageTokens(messages);
|
|
29
|
+
return tokens > this.maxContextTokens * this.compressionThreshold;
|
|
30
|
+
}
|
|
31
|
+
getTokenUsage(messages) {
|
|
32
|
+
const tokens = estimateMessageTokens(messages);
|
|
33
|
+
return {
|
|
34
|
+
tokens,
|
|
35
|
+
maxTokens: this.maxContextTokens,
|
|
36
|
+
percentage: Math.round((tokens / this.maxContextTokens) * 100),
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
truncateToolResults(messages) {
|
|
40
|
+
const maxChars = this.maxToolResultTokens * this.charsPerToken;
|
|
41
|
+
return messages.map((msg) => {
|
|
42
|
+
if (msg.role !== 'tool' || !msg.content)
|
|
43
|
+
return msg;
|
|
44
|
+
const text = messageContentToString(msg.content);
|
|
45
|
+
const tokens = estimateTokens(text);
|
|
46
|
+
if (tokens <= this.maxToolResultTokens)
|
|
47
|
+
return msg;
|
|
48
|
+
let cutoff = Math.floor(maxChars);
|
|
49
|
+
while (cutoff > 0 && text.charCodeAt(cutoff) >= 0xDC00 && text.charCodeAt(cutoff) <= 0xDFFF) {
|
|
50
|
+
cutoff--;
|
|
51
|
+
}
|
|
52
|
+
const truncated = text.slice(0, cutoff) +
|
|
53
|
+
'\n\n...[truncated — ' + (tokens - this.maxToolResultTokens) + ' tokens removed]';
|
|
54
|
+
return { ...msg, content: truncated };
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
splitForCompression(messages) {
|
|
58
|
+
if (messages.length <= this.minRecentMessages) {
|
|
59
|
+
return { toCompress: [], toKeep: [...messages] };
|
|
60
|
+
}
|
|
61
|
+
const keepCount = Math.max(this.minRecentMessages, Math.floor(messages.length * COMPRESSION_DEFAULTS.keepRatio));
|
|
62
|
+
let splitIndex = messages.length - keepCount;
|
|
63
|
+
// Walk backwards to avoid splitting tool-call/result pairs
|
|
64
|
+
while (splitIndex > 0) {
|
|
65
|
+
const msg = messages[splitIndex];
|
|
66
|
+
if (msg.role === 'tool') {
|
|
67
|
+
splitIndex--;
|
|
68
|
+
if (splitIndex > 0 && messages[splitIndex]?.role === 'assistant' && messages[splitIndex]?.tool_calls) {
|
|
69
|
+
splitIndex--;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
else {
|
|
73
|
+
break;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
splitIndex = Math.max(0, splitIndex);
|
|
77
|
+
return {
|
|
78
|
+
toCompress: messages.slice(0, splitIndex),
|
|
79
|
+
toKeep: messages.slice(splitIndex),
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
buildCompressionPrompt(messages) {
|
|
83
|
+
const maxPerMessage = 4000;
|
|
84
|
+
const formatted = messages.map((m) => {
|
|
85
|
+
let line = `[${m.role}]`;
|
|
86
|
+
if (m.name)
|
|
87
|
+
line += ` (${m.name})`;
|
|
88
|
+
if (m.content) {
|
|
89
|
+
const text = messageContentToString(m.content);
|
|
90
|
+
const display = text.length > maxPerMessage
|
|
91
|
+
? text.slice(0, maxPerMessage) + `\n...[${text.length - maxPerMessage} chars omitted]`
|
|
92
|
+
: text;
|
|
93
|
+
line += `: ${display}`;
|
|
94
|
+
}
|
|
95
|
+
if (m.tool_calls) {
|
|
96
|
+
const calls = m.tool_calls.map((tc) => {
|
|
97
|
+
const args = tc.function.arguments;
|
|
98
|
+
const preview = args.length > 200 ? args.slice(0, 200) + '...' : args;
|
|
99
|
+
return `${tc.function.name}(${preview})`;
|
|
100
|
+
});
|
|
101
|
+
line += `: called ${calls.join(', ')}`;
|
|
102
|
+
}
|
|
103
|
+
return line;
|
|
104
|
+
}).join('\n');
|
|
105
|
+
return `You are compressing a conversation to fit within the context window. The compressed output will REPLACE the original messages — anything you omit is permanently lost.
|
|
106
|
+
|
|
107
|
+
MUST PRESERVE (verbatim where possible):
|
|
108
|
+
1. The user's goal — what they are trying to accomplish and why
|
|
109
|
+
2. Every file path, function name, and line number mentioned
|
|
110
|
+
3. Every error message, stack trace, and fix applied
|
|
111
|
+
4. What was tried, what worked, what failed, and why
|
|
112
|
+
5. The user's explicit preferences, corrections, and instructions
|
|
113
|
+
6. Pending/incomplete work and next steps
|
|
114
|
+
7. Key code snippets, commands run, and their results
|
|
115
|
+
8. Configuration values, environment details, model names
|
|
116
|
+
|
|
117
|
+
DO NOT preserve greetings, acknowledgments, thinking-out-loud, or redundant explanations.
|
|
118
|
+
|
|
119
|
+
Conversation to compress:
|
|
120
|
+
${formatted}
|
|
121
|
+
|
|
122
|
+
Write a detailed, factual summary. Use specific names, paths, and values — not vague references. This summary is the agent's only memory of this work:`;
|
|
123
|
+
}
|
|
124
|
+
applySlidingWindow(messages) {
|
|
125
|
+
const usage = this.getTokenUsage(messages);
|
|
126
|
+
if (usage.percentage < 95)
|
|
127
|
+
return messages;
|
|
128
|
+
const target = this.maxContextTokens * this.slidingWindowTarget;
|
|
129
|
+
const result = [...messages];
|
|
130
|
+
while (estimateMessageTokens(result) > target && result.length > this.minRecentMessages) {
|
|
131
|
+
const removed = result.shift();
|
|
132
|
+
// Remove orphaned tool results: if we removed an assistant with tool_calls,
|
|
133
|
+
// also remove its paired tool results
|
|
134
|
+
if (removed?.role === 'assistant' && removed.tool_calls) {
|
|
135
|
+
const toolCallIds = new Set(removed.tool_calls.map((tc) => tc.id));
|
|
136
|
+
while (result.length > 0 && result[0]?.role === 'tool' && result[0]?.tool_call_id && toolCallIds.has(result[0].tool_call_id)) {
|
|
137
|
+
result.shift();
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
// If we're about to leave orphaned tool results at the front (their assistant was already removed),
|
|
141
|
+
// clean them up
|
|
142
|
+
while (result.length > 0 && result[0]?.role === 'tool') {
|
|
143
|
+
result.shift();
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return result;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
@@ -0,0 +1,257 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { estimateTokens } from './token-counter.js';
|
|
4
|
+
export const RETRIEVAL_DEFAULTS = {
|
|
5
|
+
enabled: true,
|
|
6
|
+
maxSnippets: 5,
|
|
7
|
+
maxTokensPerSnippet: 500,
|
|
8
|
+
maxTotalTokens: 3000,
|
|
9
|
+
previewLines: 30,
|
|
10
|
+
minRelevanceScore: 0.3,
|
|
11
|
+
};
|
|
12
|
+
const STOP_WORDS = new Set([
|
|
13
|
+
'a', 'an', 'the', 'is', 'are', 'was', 'were', 'be', 'been',
|
|
14
|
+
'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by', 'from',
|
|
15
|
+
'and', 'or', 'but', 'not', 'no', 'so', 'if', 'then', 'else',
|
|
16
|
+
'do', 'does', 'did', 'will', 'would', 'should', 'could', 'can',
|
|
17
|
+
'has', 'have', 'had', 'this', 'that', 'these', 'those',
|
|
18
|
+
'it', 'its', 'i', 'me', 'my', 'we', 'our', 'you', 'your',
|
|
19
|
+
'he', 'she', 'they', 'them', 'what', 'which', 'who', 'how',
|
|
20
|
+
'when', 'where', 'why', 'all', 'each', 'every', 'any', 'some',
|
|
21
|
+
'about', 'up', 'out', 'just', 'also', 'than', 'very',
|
|
22
|
+
'code', 'file', 'function', 'change', 'make', 'use', 'using',
|
|
23
|
+
'want', 'need', 'like', 'please', 'help', 'show', 'get',
|
|
24
|
+
]);
|
|
25
|
+
const FILE_HINT_RE = /(?:^|[\s`"'(])([.\w][\w./-]*\/[\w./-]+\.\w{1,10})(?=[\s`"'),.:;]|$)/gm;
|
|
26
|
+
const CAMEL_RE = /\b([A-Z][a-zA-Z]{2,})\b/g;
|
|
27
|
+
const SNAKE_RE = /\b([a-z][a-z0-9]*(?:_[a-z0-9]+){1,})\b/g;
|
|
28
|
+
const INTENT_PATTERNS = {
|
|
29
|
+
debug: /\b(?:fix|debug|error|bug|broken|crash|fail|issue|wrong|problem)\b/i,
|
|
30
|
+
create: /\b(?:create|add|new|implement|build|generate|write|introduce)\b/i,
|
|
31
|
+
modify: /\b(?:change|update|modify|refactor|rename|move|replace|improve|optimize)\b/i,
|
|
32
|
+
explore: /\b(?:read|show|explain|understand|find|search|look|what|where|how|list|describe)\b/i,
|
|
33
|
+
general: /./,
|
|
34
|
+
};
|
|
35
|
+
export function analyzeQuery(query) {
|
|
36
|
+
const words = query.toLowerCase().split(/\s+/).filter(Boolean);
|
|
37
|
+
const keywords = words.filter((w) => !STOP_WORDS.has(w) && w.length > 2);
|
|
38
|
+
const fileHints = [];
|
|
39
|
+
for (const m of query.matchAll(FILE_HINT_RE)) {
|
|
40
|
+
fileHints.push(m[1]);
|
|
41
|
+
}
|
|
42
|
+
const symbolHints = [];
|
|
43
|
+
for (const m of query.matchAll(CAMEL_RE)) {
|
|
44
|
+
symbolHints.push(m[1]);
|
|
45
|
+
}
|
|
46
|
+
for (const m of query.matchAll(SNAKE_RE)) {
|
|
47
|
+
symbolHints.push(m[1]);
|
|
48
|
+
}
|
|
49
|
+
let intentType = 'general';
|
|
50
|
+
for (const [intent, re] of Object.entries(INTENT_PATTERNS)) {
|
|
51
|
+
if (intent === 'general')
|
|
52
|
+
continue;
|
|
53
|
+
if (re.test(query)) {
|
|
54
|
+
intentType = intent;
|
|
55
|
+
break;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
return { keywords, fileHints, symbolHints, intentType };
|
|
59
|
+
}
|
|
60
|
+
function scoreFiles(analysis, index) {
|
|
61
|
+
const symbolsByFile = new Map();
|
|
62
|
+
for (const sym of index.symbols) {
|
|
63
|
+
const existing = symbolsByFile.get(sym.file) ?? [];
|
|
64
|
+
existing.push(sym);
|
|
65
|
+
symbolsByFile.set(sym.file, existing);
|
|
66
|
+
}
|
|
67
|
+
const importGraph = new Map();
|
|
68
|
+
for (const imp of index.imports) {
|
|
69
|
+
const deps = importGraph.get(imp.from) ?? new Set();
|
|
70
|
+
deps.add(imp.to);
|
|
71
|
+
importGraph.set(imp.from, deps);
|
|
72
|
+
}
|
|
73
|
+
const recentFiles = new Set(index.files
|
|
74
|
+
.sort((a, b) => b.modifiedAt - a.modifiedAt)
|
|
75
|
+
.slice(0, 20)
|
|
76
|
+
.map((f) => f.relativePath));
|
|
77
|
+
const directMentionFiles = new Set();
|
|
78
|
+
for (const hint of analysis.fileHints) {
|
|
79
|
+
for (const f of index.files) {
|
|
80
|
+
if (f.relativePath === hint || f.relativePath.endsWith(hint)) {
|
|
81
|
+
directMentionFiles.add(f.relativePath);
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
const scores = [];
|
|
86
|
+
for (const file of index.files) {
|
|
87
|
+
let score = 0;
|
|
88
|
+
const reasons = [];
|
|
89
|
+
const matchingSymbols = [];
|
|
90
|
+
// Direct file mention
|
|
91
|
+
if (directMentionFiles.has(file.relativePath)) {
|
|
92
|
+
score += 1.0;
|
|
93
|
+
reasons.push('directly mentioned');
|
|
94
|
+
}
|
|
95
|
+
// Symbol match
|
|
96
|
+
const fileSymbols = symbolsByFile.get(file.relativePath) ?? [];
|
|
97
|
+
for (const sym of fileSymbols) {
|
|
98
|
+
const nameLower = sym.name.toLowerCase();
|
|
99
|
+
if (analysis.symbolHints.some((h) => h.toLowerCase() === nameLower)) {
|
|
100
|
+
score += 0.6;
|
|
101
|
+
matchingSymbols.push(sym.name);
|
|
102
|
+
reasons.push(`symbol "${sym.name}"`);
|
|
103
|
+
}
|
|
104
|
+
else if (analysis.keywords.some((k) => nameLower.includes(k))) {
|
|
105
|
+
score += 0.3;
|
|
106
|
+
matchingSymbols.push(sym.name);
|
|
107
|
+
reasons.push(`keyword in symbol "${sym.name}"`);
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// Import proximity — file is imported by a directly-mentioned file
|
|
111
|
+
for (const mentionedFile of directMentionFiles) {
|
|
112
|
+
const deps = importGraph.get(mentionedFile);
|
|
113
|
+
if (deps?.has(file.relativePath)) {
|
|
114
|
+
score += 0.3;
|
|
115
|
+
reasons.push(`imported by ${mentionedFile}`);
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// Keyword in path
|
|
119
|
+
const pathLower = file.relativePath.toLowerCase();
|
|
120
|
+
for (const kw of analysis.keywords) {
|
|
121
|
+
if (pathLower.includes(kw)) {
|
|
122
|
+
score += 0.2;
|
|
123
|
+
reasons.push(`keyword "${kw}" in path`);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
// Recency bonus
|
|
127
|
+
if (recentFiles.has(file.relativePath)) {
|
|
128
|
+
score += 0.1;
|
|
129
|
+
reasons.push('recently modified');
|
|
130
|
+
}
|
|
131
|
+
if (score > 0) {
|
|
132
|
+
scores.push({
|
|
133
|
+
file: file.relativePath,
|
|
134
|
+
score: Math.min(score, 2.0),
|
|
135
|
+
matchingSymbols,
|
|
136
|
+
reason: reasons.join(', '),
|
|
137
|
+
});
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return scores.sort((a, b) => b.score - a.score);
|
|
141
|
+
}
|
|
142
|
+
async function generatePreview(filePath, projectRoot, matchingSymbols, maxLines) {
|
|
143
|
+
try {
|
|
144
|
+
const absPath = path.join(projectRoot, filePath);
|
|
145
|
+
const content = await fs.readFile(absPath, 'utf-8');
|
|
146
|
+
const lines = content.split('\n');
|
|
147
|
+
if (matchingSymbols.length > 0) {
|
|
148
|
+
// Show lines around matching symbols
|
|
149
|
+
const ranges = [];
|
|
150
|
+
for (const sym of matchingSymbols.slice(0, 3)) {
|
|
151
|
+
const start = Math.max(0, sym.line - 2);
|
|
152
|
+
const end = Math.min(lines.length - 1, sym.line + 8);
|
|
153
|
+
ranges.push([start, end]);
|
|
154
|
+
}
|
|
155
|
+
// Merge overlapping ranges
|
|
156
|
+
ranges.sort((a, b) => a[0] - b[0]);
|
|
157
|
+
const merged = [ranges[0]];
|
|
158
|
+
for (let i = 1; i < ranges.length; i++) {
|
|
159
|
+
const prev = merged[merged.length - 1];
|
|
160
|
+
if (ranges[i][0] <= prev[1] + 2) {
|
|
161
|
+
prev[1] = Math.max(prev[1], ranges[i][1]);
|
|
162
|
+
}
|
|
163
|
+
else {
|
|
164
|
+
merged.push(ranges[i]);
|
|
165
|
+
}
|
|
166
|
+
}
|
|
167
|
+
const parts = [];
|
|
168
|
+
let totalLines = 0;
|
|
169
|
+
for (const [start, end] of merged) {
|
|
170
|
+
if (totalLines >= maxLines)
|
|
171
|
+
break;
|
|
172
|
+
const available = maxLines - totalLines;
|
|
173
|
+
const rangeEnd = Math.min(end, start + available - 1);
|
|
174
|
+
for (let i = start; i <= rangeEnd; i++) {
|
|
175
|
+
parts.push(`${i + 1}\t${lines[i]}`);
|
|
176
|
+
totalLines++;
|
|
177
|
+
}
|
|
178
|
+
if (rangeEnd < end)
|
|
179
|
+
break;
|
|
180
|
+
parts.push('...');
|
|
181
|
+
}
|
|
182
|
+
return parts.join('\n');
|
|
183
|
+
}
|
|
184
|
+
// No matching symbols — show first N lines
|
|
185
|
+
return lines
|
|
186
|
+
.slice(0, maxLines)
|
|
187
|
+
.map((l, i) => `${i + 1}\t${l}`)
|
|
188
|
+
.join('\n');
|
|
189
|
+
}
|
|
190
|
+
catch {
|
|
191
|
+
return '(file could not be read)';
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
export async function retrieveContext(query, index, config = RETRIEVAL_DEFAULTS) {
|
|
195
|
+
const analysis = analyzeQuery(query);
|
|
196
|
+
if (analysis.keywords.length === 0 && analysis.fileHints.length === 0 && analysis.symbolHints.length === 0) {
|
|
197
|
+
return { snippets: [], totalTokens: 0, queryTerms: [] };
|
|
198
|
+
}
|
|
199
|
+
const scored = scoreFiles(analysis, index);
|
|
200
|
+
const candidates = scored.filter((s) => s.score >= config.minRelevanceScore).slice(0, config.maxSnippets);
|
|
201
|
+
const symbolsByFile = new Map();
|
|
202
|
+
for (const sym of index.symbols) {
|
|
203
|
+
const existing = symbolsByFile.get(sym.file) ?? [];
|
|
204
|
+
existing.push(sym);
|
|
205
|
+
symbolsByFile.set(sym.file, existing);
|
|
206
|
+
}
|
|
207
|
+
const snippets = [];
|
|
208
|
+
let totalTokens = 0;
|
|
209
|
+
for (const candidate of candidates) {
|
|
210
|
+
if (totalTokens >= config.maxTotalTokens)
|
|
211
|
+
break;
|
|
212
|
+
const matchingSymbols = (symbolsByFile.get(candidate.file) ?? [])
|
|
213
|
+
.filter((s) => candidate.matchingSymbols.includes(s.name));
|
|
214
|
+
const preview = await generatePreview(candidate.file, index.root, matchingSymbols, config.previewLines);
|
|
215
|
+
let tokens = estimateTokens(preview);
|
|
216
|
+
let truncatedPreview = preview;
|
|
217
|
+
if (tokens > config.maxTokensPerSnippet) {
|
|
218
|
+
const lines = preview.split('\n');
|
|
219
|
+
const kept = [];
|
|
220
|
+
let running = 0;
|
|
221
|
+
for (const line of lines) {
|
|
222
|
+
const lt = estimateTokens(line);
|
|
223
|
+
if (running + lt > config.maxTokensPerSnippet)
|
|
224
|
+
break;
|
|
225
|
+
kept.push(line);
|
|
226
|
+
running += lt;
|
|
227
|
+
}
|
|
228
|
+
truncatedPreview = kept.join('\n');
|
|
229
|
+
tokens = running;
|
|
230
|
+
}
|
|
231
|
+
if (totalTokens + tokens > config.maxTotalTokens)
|
|
232
|
+
break;
|
|
233
|
+
snippets.push({
|
|
234
|
+
file: candidate.file,
|
|
235
|
+
relevanceScore: candidate.score,
|
|
236
|
+
symbols: candidate.matchingSymbols,
|
|
237
|
+
preview: truncatedPreview,
|
|
238
|
+
reason: candidate.reason,
|
|
239
|
+
});
|
|
240
|
+
totalTokens += tokens;
|
|
241
|
+
}
|
|
242
|
+
return { snippets, totalTokens, queryTerms: analysis.keywords };
|
|
243
|
+
}
|
|
244
|
+
export function snippetsToContext(snippets) {
|
|
245
|
+
if (snippets.length === 0)
|
|
246
|
+
return '';
|
|
247
|
+
const parts = ['## Relevant Code Context',
|
|
248
|
+
'The following file excerpts may be relevant. Use this as ground truth rather than guessing.\n'];
|
|
249
|
+
for (const s of snippets) {
|
|
250
|
+
const symbolStr = s.symbols.length > 0 ? `\nSymbols: ${s.symbols.join(', ')}` : '';
|
|
251
|
+
parts.push(`### ${s.file} (score: ${s.relevanceScore.toFixed(1)})${symbolStr}`);
|
|
252
|
+
parts.push('```');
|
|
253
|
+
parts.push(s.preview);
|
|
254
|
+
parts.push('```\n');
|
|
255
|
+
}
|
|
256
|
+
return parts.join('\n');
|
|
257
|
+
}
|
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
export class CostTracker {
|
|
2
|
+
entries = [];
|
|
3
|
+
sessionStart = Date.now();
|
|
4
|
+
record(model, usage) {
|
|
5
|
+
this.entries.push({
|
|
6
|
+
model,
|
|
7
|
+
promptTokens: usage.prompt_tokens ?? 0,
|
|
8
|
+
completionTokens: usage.completion_tokens ?? 0,
|
|
9
|
+
totalTokens: usage.total_tokens ?? 0,
|
|
10
|
+
cost: usage.cost ?? 0,
|
|
11
|
+
timestamp: Date.now(),
|
|
12
|
+
});
|
|
13
|
+
}
|
|
14
|
+
get totalTokens() {
|
|
15
|
+
return this.entries.reduce((s, e) => s + e.totalTokens, 0);
|
|
16
|
+
}
|
|
17
|
+
get totalCost() {
|
|
18
|
+
return this.entries.reduce((s, e) => s + e.cost, 0);
|
|
19
|
+
}
|
|
20
|
+
get promptTokens() {
|
|
21
|
+
return this.entries.reduce((s, e) => s + e.promptTokens, 0);
|
|
22
|
+
}
|
|
23
|
+
get completionTokens() {
|
|
24
|
+
return this.entries.reduce((s, e) => s + e.completionTokens, 0);
|
|
25
|
+
}
|
|
26
|
+
get requestCount() {
|
|
27
|
+
return this.entries.length;
|
|
28
|
+
}
|
|
29
|
+
get sessionDuration() {
|
|
30
|
+
const s = Math.floor((Date.now() - this.sessionStart) / 1000);
|
|
31
|
+
if (s < 60)
|
|
32
|
+
return `${s}s`;
|
|
33
|
+
if (s < 3600)
|
|
34
|
+
return `${Math.floor(s / 60)}m${s % 60}s`;
|
|
35
|
+
return `${Math.floor(s / 3600)}h${Math.floor((s % 3600) / 60)}m`;
|
|
36
|
+
}
|
|
37
|
+
summary() {
|
|
38
|
+
const lines = [
|
|
39
|
+
`Session: ${this.sessionDuration} | ${this.requestCount} requests`,
|
|
40
|
+
`Tokens: ${this.totalTokens.toLocaleString()} (${this.promptTokens.toLocaleString()} prompt + ${this.completionTokens.toLocaleString()} completion)`,
|
|
41
|
+
];
|
|
42
|
+
const byModel = new Map();
|
|
43
|
+
for (const e of this.entries) {
|
|
44
|
+
if (e.model === '(previous session)')
|
|
45
|
+
continue;
|
|
46
|
+
const prev = byModel.get(e.model) ?? { tokens: 0, count: 0 };
|
|
47
|
+
byModel.set(e.model, { tokens: prev.tokens + e.totalTokens, count: prev.count + 1 });
|
|
48
|
+
}
|
|
49
|
+
if (byModel.size > 0) {
|
|
50
|
+
lines.push('By model:');
|
|
51
|
+
for (const [model, stats] of byModel) {
|
|
52
|
+
const short = model.split('/').pop()?.replace(/:free$/, '') ?? model;
|
|
53
|
+
lines.push(` ${short}: ${stats.tokens.toLocaleString()} tokens (${stats.count} reqs)`);
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
return lines.join('\n');
|
|
57
|
+
}
|
|
58
|
+
reset() {
|
|
59
|
+
this.entries = [];
|
|
60
|
+
this.sessionStart = Date.now();
|
|
61
|
+
}
|
|
62
|
+
restore(state) {
|
|
63
|
+
this.entries = [];
|
|
64
|
+
if (state.requestCount > 0 && state.totalTokens >= 0) {
|
|
65
|
+
const count = Math.max(1, state.requestCount);
|
|
66
|
+
const perReqTokens = Math.floor(state.totalTokens / count);
|
|
67
|
+
const remainder = state.totalTokens - perReqTokens * count;
|
|
68
|
+
const perReqCost = state.totalCost / count;
|
|
69
|
+
for (let i = 0; i < count; i++) {
|
|
70
|
+
this.entries.push({
|
|
71
|
+
model: '(previous session)',
|
|
72
|
+
promptTokens: 0,
|
|
73
|
+
completionTokens: 0,
|
|
74
|
+
totalTokens: perReqTokens + (i === 0 ? remainder : 0),
|
|
75
|
+
cost: perReqCost,
|
|
76
|
+
timestamp: Date.now(),
|
|
77
|
+
});
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
getState() {
|
|
82
|
+
return { totalTokens: this.totalTokens, totalCost: this.totalCost, requestCount: this.requestCount };
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
export const costTracker = new CostTracker();
|
package/dist/diff.js
ADDED
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
function lcsLengths(a, b) {
|
|
2
|
+
const m = a.length;
|
|
3
|
+
const n = b.length;
|
|
4
|
+
const dp = Array.from({ length: m + 1 }, () => new Array(n + 1).fill(0));
|
|
5
|
+
for (let i = 1; i <= m; i++) {
|
|
6
|
+
for (let j = 1; j <= n; j++) {
|
|
7
|
+
dp[i][j] = a[i - 1] === b[j - 1] ? dp[i - 1][j - 1] + 1 : Math.max(dp[i - 1][j], dp[i][j - 1]);
|
|
8
|
+
}
|
|
9
|
+
}
|
|
10
|
+
return dp;
|
|
11
|
+
}
|
|
12
|
+
function backtrack(dp, a, b) {
|
|
13
|
+
const lines = [];
|
|
14
|
+
let i = a.length;
|
|
15
|
+
let j = b.length;
|
|
16
|
+
while (i > 0 || j > 0) {
|
|
17
|
+
if (i > 0 && j > 0 && a[i - 1] === b[j - 1]) {
|
|
18
|
+
lines.push({ type: 'context', lineOld: i, lineNew: j, text: a[i - 1] });
|
|
19
|
+
i--;
|
|
20
|
+
j--;
|
|
21
|
+
}
|
|
22
|
+
else if (j > 0 && (i === 0 || dp[i][j - 1] > dp[i - 1][j])) {
|
|
23
|
+
lines.push({ type: 'add', lineNew: j, text: b[j - 1] });
|
|
24
|
+
j--;
|
|
25
|
+
}
|
|
26
|
+
else {
|
|
27
|
+
lines.push({ type: 'remove', lineOld: i, text: a[i - 1] });
|
|
28
|
+
i--;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return lines.reverse();
|
|
32
|
+
}
|
|
33
|
+
const LARGE_FILE_THRESHOLD = 50_000;
|
|
34
|
+
function simpleDiff(oldLines, newLines) {
|
|
35
|
+
const lines = [];
|
|
36
|
+
const oldSet = new Map();
|
|
37
|
+
oldLines.forEach((l, i) => {
|
|
38
|
+
if (!oldSet.has(l))
|
|
39
|
+
oldSet.set(l, []);
|
|
40
|
+
oldSet.get(l).push(i + 1);
|
|
41
|
+
});
|
|
42
|
+
let commonPrefix = 0;
|
|
43
|
+
while (commonPrefix < oldLines.length && commonPrefix < newLines.length && oldLines[commonPrefix] === newLines[commonPrefix]) {
|
|
44
|
+
lines.push({ type: 'context', lineOld: commonPrefix + 1, lineNew: commonPrefix + 1, text: oldLines[commonPrefix] });
|
|
45
|
+
commonPrefix++;
|
|
46
|
+
}
|
|
47
|
+
let commonSuffix = 0;
|
|
48
|
+
while (commonSuffix < oldLines.length - commonPrefix &&
|
|
49
|
+
commonSuffix < newLines.length - commonPrefix &&
|
|
50
|
+
oldLines[oldLines.length - 1 - commonSuffix] === newLines[newLines.length - 1 - commonSuffix]) {
|
|
51
|
+
commonSuffix++;
|
|
52
|
+
}
|
|
53
|
+
for (let i = commonPrefix; i < oldLines.length - commonSuffix; i++) {
|
|
54
|
+
lines.push({ type: 'remove', lineOld: i + 1, text: oldLines[i] });
|
|
55
|
+
}
|
|
56
|
+
for (let i = commonPrefix; i < newLines.length - commonSuffix; i++) {
|
|
57
|
+
lines.push({ type: 'add', lineNew: i + 1, text: newLines[i] });
|
|
58
|
+
}
|
|
59
|
+
for (let s = commonSuffix - 1; s >= 0; s--) {
|
|
60
|
+
const oldIdx = oldLines.length - 1 - s;
|
|
61
|
+
const newIdx = newLines.length - 1 - s;
|
|
62
|
+
lines.push({ type: 'context', lineOld: oldIdx + 1, lineNew: newIdx + 1, text: oldLines[oldIdx] });
|
|
63
|
+
}
|
|
64
|
+
return lines;
|
|
65
|
+
}
|
|
66
|
+
export function computeDiff(oldText, newText) {
|
|
67
|
+
if (oldText === newText)
|
|
68
|
+
return { hunks: [], additions: 0, deletions: 0 };
|
|
69
|
+
const oldLines = oldText === '' ? [] : oldText.split('\n');
|
|
70
|
+
const newLines = newText === '' ? [] : newText.split('\n');
|
|
71
|
+
const isLarge = oldLines.length > LARGE_FILE_THRESHOLD || newLines.length > LARGE_FILE_THRESHOLD;
|
|
72
|
+
const allLines = isLarge
|
|
73
|
+
? simpleDiff(oldLines, newLines)
|
|
74
|
+
: backtrack(lcsLengths(oldLines, newLines), oldLines, newLines);
|
|
75
|
+
const hunks = [];
|
|
76
|
+
let currentHunk = [];
|
|
77
|
+
let contextCount = 0;
|
|
78
|
+
const CONTEXT_SIZE = 3;
|
|
79
|
+
for (let idx = 0; idx < allLines.length; idx++) {
|
|
80
|
+
const line = allLines[idx];
|
|
81
|
+
if (line.type === 'context') {
|
|
82
|
+
contextCount++;
|
|
83
|
+
if (currentHunk.length > 0 && contextCount > CONTEXT_SIZE * 2) {
|
|
84
|
+
hunks.push(currentHunk);
|
|
85
|
+
currentHunk = [];
|
|
86
|
+
contextCount = 1;
|
|
87
|
+
}
|
|
88
|
+
if (currentHunk.length > 0 || contextCount <= CONTEXT_SIZE) {
|
|
89
|
+
currentHunk.push(line);
|
|
90
|
+
}
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
if (contextCount > CONTEXT_SIZE && currentHunk.length === 0) {
|
|
94
|
+
for (let k = Math.max(0, idx - CONTEXT_SIZE); k < idx; k++) {
|
|
95
|
+
if (allLines[k].type === 'context')
|
|
96
|
+
currentHunk.push(allLines[k]);
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
currentHunk.push(line);
|
|
100
|
+
contextCount = 0;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
if (currentHunk.some((l) => l.type !== 'context')) {
|
|
104
|
+
hunks.push(currentHunk);
|
|
105
|
+
}
|
|
106
|
+
const additions = allLines.filter((l) => l.type === 'add').length;
|
|
107
|
+
const deletions = allLines.filter((l) => l.type === 'remove').length;
|
|
108
|
+
return { hunks, additions, deletions };
|
|
109
|
+
}
|
|
110
|
+
export function formatDiff(diff, filePath) {
|
|
111
|
+
if (diff.hunks.length === 0)
|
|
112
|
+
return '(no changes)';
|
|
113
|
+
const lines = [];
|
|
114
|
+
if (filePath) {
|
|
115
|
+
lines.push(`--- ${filePath}`);
|
|
116
|
+
lines.push(`+++ ${filePath}`);
|
|
117
|
+
}
|
|
118
|
+
for (const hunk of diff.hunks) {
|
|
119
|
+
const firstOld = hunk.find((l) => l.lineOld)?.lineOld ?? 1;
|
|
120
|
+
const firstNew = hunk.find((l) => l.lineNew)?.lineNew ?? 1;
|
|
121
|
+
const oldCount = hunk.filter((l) => l.type !== 'add').length;
|
|
122
|
+
const newCount = hunk.filter((l) => l.type !== 'remove').length;
|
|
123
|
+
lines.push(`@@ -${firstOld},${oldCount} +${firstNew},${newCount} @@`);
|
|
124
|
+
for (const line of hunk) {
|
|
125
|
+
switch (line.type) {
|
|
126
|
+
case 'add':
|
|
127
|
+
lines.push(`\x1b[32m+ ${line.text}\x1b[0m`);
|
|
128
|
+
break;
|
|
129
|
+
case 'remove':
|
|
130
|
+
lines.push(`\x1b[31m- ${line.text}\x1b[0m`);
|
|
131
|
+
break;
|
|
132
|
+
case 'context':
|
|
133
|
+
lines.push(` ${line.text}`);
|
|
134
|
+
break;
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
lines.push(`\x1b[36m${diff.additions} additions, ${diff.deletions} deletions\x1b[0m`);
|
|
139
|
+
return lines.join('\n');
|
|
140
|
+
}
|