smart-context-mcp 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +414 -0
- package/package.json +63 -0
- package/scripts/devctx-server.js +4 -0
- package/scripts/init-clients.js +356 -0
- package/scripts/report-metrics.js +195 -0
- package/src/index.js +976 -0
- package/src/mcp-server.js +3 -0
- package/src/metrics.js +65 -0
- package/src/server.js +143 -0
- package/src/tokenCounter.js +12 -0
- package/src/tools/smart-context.js +1192 -0
- package/src/tools/smart-read/additional-languages.js +684 -0
- package/src/tools/smart-read/code.js +216 -0
- package/src/tools/smart-read/fallback.js +23 -0
- package/src/tools/smart-read/python.js +178 -0
- package/src/tools/smart-read/shared.js +39 -0
- package/src/tools/smart-read/structured.js +72 -0
- package/src/tools/smart-read-batch.js +63 -0
- package/src/tools/smart-read.js +459 -0
- package/src/tools/smart-search.js +412 -0
- package/src/tools/smart-shell.js +213 -0
- package/src/utils/fs.js +47 -0
- package/src/utils/paths.js +1 -0
- package/src/utils/runtime-config.js +29 -0
- package/src/utils/text.js +38 -0
|
@@ -0,0 +1,412 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { execFile as execFileCallback } from 'node:child_process';
|
|
4
|
+
import { promisify } from 'node:util';
|
|
5
|
+
import { rgPath } from '@vscode/ripgrep';
|
|
6
|
+
import { buildMetrics, persistMetrics } from '../metrics.js';
|
|
7
|
+
import { loadIndex, queryIndex, queryRelated } from '../index.js';
|
|
8
|
+
import { projectRoot } from '../utils/paths.js';
|
|
9
|
+
import { isBinaryBuffer, isDockerfile, resolveSafePath } from '../utils/fs.js';
|
|
10
|
+
import { truncate } from '../utils/text.js';
|
|
11
|
+
|
|
12
|
+
const execFile = promisify(execFileCallback);
|
|
13
|
+
const supportedGlobs = [
|
|
14
|
+
'*.js', '*.jsx', '*.ts', '*.tsx', '*.json', '*.mjs', '*.cjs',
|
|
15
|
+
'*.py', '*.toml', '*.yaml', '*.yml', '*.md', '*.graphql', '*.gql', '*.sql',
|
|
16
|
+
'*.go', '*.rs', '*.java', '*.sh', '*.bash', '*.zsh', '*.tf', '*.tfvars', '*.hcl',
|
|
17
|
+
'Dockerfile', 'Dockerfile.*',
|
|
18
|
+
];
|
|
19
|
+
const ignoredDirs = ['node_modules', '.git', '.next', 'dist', 'build', 'coverage', '.venv', 'venv', '__pycache__', '.terraform'];
|
|
20
|
+
const ignoredFileNames = new Set(['pnpm-lock.yaml', 'package-lock.json', 'yarn.lock', 'bun.lockb', 'npm-shrinkwrap.json']);
|
|
21
|
+
const fallbackExtensions = new Set(['.js', '.jsx', '.ts', '.tsx', '.json', '.mjs', '.cjs', '.py', '.toml', '.yaml', '.yml', '.md', '.graphql', '.gql', '.sql', '.go', '.rs', '.java', '.sh', '.bash', '.zsh', '.tf', '.tfvars', '.hcl']);
|
|
22
|
+
const likelySourceExtensions = new Set(['.js', '.jsx', '.ts', '.tsx', '.py', '.graphql', '.gql', '.sql', '.go', '.rs', '.java', '.sh', '.bash', '.zsh']);
|
|
23
|
+
const likelyConfigExtensions = new Set(['.json', '.toml', '.yaml', '.yml', '.tf', '.tfvars', '.hcl']);
|
|
24
|
+
const lowSignalNames = ['changelog', 'readme', 'migration', 'license', 'licence', 'contributing', 'authors', 'code_of_conduct', 'security', 'history'];
|
|
25
|
+
const testPatterns = ['.test.', '.spec.', '__tests__', '__mocks__', 'fixtures'];
|
|
26
|
+
|
|
27
|
+
export const VALID_INTENTS = new Set(['implementation', 'debug', 'tests', 'config', 'docs', 'explore']);
|
|
28
|
+
|
|
29
|
+
export const intentWeights = {
|
|
30
|
+
implementation: { src: 10, source: 14, config: 4, lowSignal: -35, test: -15 },
|
|
31
|
+
debug: { src: 10, source: 14, config: 4, lowSignal: -35, test: -15 },
|
|
32
|
+
tests: { src: 5, source: 10, config: 0, lowSignal: -35, test: 10 },
|
|
33
|
+
config: { src: 0, source: 0, config: 14, lowSignal: -20, test: -15 },
|
|
34
|
+
docs: { src: 0, source: 4, config: 4, lowSignal: -10, test: -15 },
|
|
35
|
+
explore: { src: 10, source: 14, config: 4, lowSignal: -35, test: -15 },
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const defaultWeights = intentWeights.explore;
|
|
39
|
+
|
|
40
|
+
const shouldIgnoreFile = (filePath) => ignoredFileNames.has(path.basename(filePath));
|
|
41
|
+
|
|
42
|
+
const isSearchableFile = (entryName, fullPath) => fallbackExtensions.has(path.extname(entryName)) || isDockerfile(fullPath);
|
|
43
|
+
|
|
44
|
+
export const walk = (dir, files = [], stats = { skippedDirs: 0 }) => {
|
|
45
|
+
let entries;
|
|
46
|
+
try {
|
|
47
|
+
entries = fs.readdirSync(dir, { withFileTypes: true });
|
|
48
|
+
} catch {
|
|
49
|
+
stats.skippedDirs++;
|
|
50
|
+
return files;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
for (const entry of entries) {
|
|
54
|
+
if (ignoredDirs.includes(entry.name) || ignoredFileNames.has(entry.name)) {
|
|
55
|
+
continue;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const fullPath = path.join(dir, entry.name);
|
|
59
|
+
|
|
60
|
+
if (entry.isDirectory()) {
|
|
61
|
+
walk(fullPath, files, stats);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
if (isSearchableFile(entry.name, fullPath)) {
|
|
66
|
+
files.push(fullPath);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
return files;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
const parseRgLine = (line, root) => {
|
|
74
|
+
const match = /^(.*?):(\d+):(.*)$/.exec(line);
|
|
75
|
+
|
|
76
|
+
if (!match) {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const [, relativePath, lineNumber, content] = match;
|
|
81
|
+
return {
|
|
82
|
+
file: path.join(root, relativePath),
|
|
83
|
+
lineNumber: Number(lineNumber),
|
|
84
|
+
content,
|
|
85
|
+
};
|
|
86
|
+
};
|
|
87
|
+
|
|
88
|
+
const searchWithRipgrep = async (root, query) => {
|
|
89
|
+
const args = [
|
|
90
|
+
'--line-number',
|
|
91
|
+
'--no-heading',
|
|
92
|
+
'--color',
|
|
93
|
+
'never',
|
|
94
|
+
'--smart-case',
|
|
95
|
+
'--fixed-strings',
|
|
96
|
+
];
|
|
97
|
+
|
|
98
|
+
for (const dir of ignoredDirs) {
|
|
99
|
+
args.push('--glob', `!${dir}/**`);
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
for (const fileName of ignoredFileNames) {
|
|
103
|
+
args.push('--glob', `!${fileName}`);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
for (const extension of supportedGlobs) {
|
|
107
|
+
args.push('--glob', extension);
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
args.push(query, '.');
|
|
111
|
+
|
|
112
|
+
try {
|
|
113
|
+
const { stdout } = await execFile(rgPath, args, {
|
|
114
|
+
cwd: root,
|
|
115
|
+
maxBuffer: 1024 * 1024 * 10,
|
|
116
|
+
timeout: 15000,
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
return stdout
|
|
120
|
+
.split('\n')
|
|
121
|
+
.filter(Boolean)
|
|
122
|
+
.map((line) => parseRgLine(line, root))
|
|
123
|
+
.filter(Boolean)
|
|
124
|
+
.filter((match) => !shouldIgnoreFile(match.file));
|
|
125
|
+
} catch (error) {
|
|
126
|
+
if (error.code === 1) return [];
|
|
127
|
+
return null;
|
|
128
|
+
}
|
|
129
|
+
};
|
|
130
|
+
|
|
131
|
+
const MAX_FALLBACK_FILE_BYTES = 1024 * 1024;
|
|
132
|
+
|
|
133
|
+
export const isSmartCaseSensitive = (query) => query !== query.toLowerCase();
|
|
134
|
+
|
|
135
|
+
export const searchWithFallback = (root, query) => {
|
|
136
|
+
const walkStats = { skippedDirs: 0 };
|
|
137
|
+
const files = walk(root, [], walkStats);
|
|
138
|
+
const matches = [];
|
|
139
|
+
const caseSensitive = isSmartCaseSensitive(query);
|
|
140
|
+
const comparator = caseSensitive
|
|
141
|
+
? (line) => line.includes(query)
|
|
142
|
+
: (line) => line.toLowerCase().includes(query.toLowerCase());
|
|
143
|
+
let skippedLarge = 0;
|
|
144
|
+
let skippedBinary = 0;
|
|
145
|
+
let skippedErrors = 0;
|
|
146
|
+
|
|
147
|
+
for (const file of files) {
|
|
148
|
+
try {
|
|
149
|
+
const stat = fs.statSync(file);
|
|
150
|
+
if (stat.size > MAX_FALLBACK_FILE_BYTES) { skippedLarge++; continue; }
|
|
151
|
+
|
|
152
|
+
const buffer = fs.readFileSync(file);
|
|
153
|
+
if (isBinaryBuffer(buffer)) { skippedBinary++; continue; }
|
|
154
|
+
|
|
155
|
+
const content = buffer.toString('utf8');
|
|
156
|
+
const lines = content.split('\n');
|
|
157
|
+
|
|
158
|
+
lines.forEach((line, index) => {
|
|
159
|
+
if (comparator(line)) {
|
|
160
|
+
matches.push({
|
|
161
|
+
file,
|
|
162
|
+
lineNumber: index + 1,
|
|
163
|
+
content: line,
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
});
|
|
167
|
+
} catch {
|
|
168
|
+
skippedErrors++;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return { matches, caseSensitive, skippedLarge, skippedBinary, skippedErrors, skippedDirs: walkStats.skippedDirs };
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
const formatMatch = (match) => `${match.file}:${match.lineNumber}:${match.content}`;
|
|
176
|
+
|
|
177
|
+
const scoreGroup = (group, query, intent) => {
|
|
178
|
+
const w = (intent && intentWeights[intent]) || defaultWeights;
|
|
179
|
+
const normalizedQuery = query.toLowerCase();
|
|
180
|
+
const lowerFilePath = group.file.toLowerCase();
|
|
181
|
+
const fileName = path.basename(group.file).toLowerCase();
|
|
182
|
+
const extension = path.extname(group.file).toLowerCase();
|
|
183
|
+
const pathDepth = group.file.split(path.sep).length;
|
|
184
|
+
const sampleText = group.matches.slice(0, 5).map((match) => match.content.toLowerCase()).join(' ');
|
|
185
|
+
const pathSegments = lowerFilePath.split(/[\\/._-]+/).filter(Boolean);
|
|
186
|
+
let score = Math.min(group.count, 12) * 6;
|
|
187
|
+
|
|
188
|
+
if (fileName.includes(normalizedQuery)) {
|
|
189
|
+
score += 30;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
if (pathSegments.includes(normalizedQuery)) {
|
|
193
|
+
score += 16;
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
if (lowerFilePath.includes(`${path.sep}src${path.sep}`)) {
|
|
197
|
+
score += w.src;
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
if (lowerFilePath.includes(`${path.sep}packages${path.sep}`) || lowerFilePath.includes(`${path.sep}apps${path.sep}`)) {
|
|
201
|
+
score += 8;
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
if (likelySourceExtensions.has(extension) || isDockerfile(group.file)) {
|
|
205
|
+
score += w.source;
|
|
206
|
+
} else if (likelyConfigExtensions.has(extension)) {
|
|
207
|
+
score += w.config;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
if (sampleText.includes(normalizedQuery)) {
|
|
211
|
+
score += 8;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
if (lowSignalNames.some((name) => fileName.includes(name))) {
|
|
215
|
+
score += w.lowSignal;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
if (testPatterns.some((p) => lowerFilePath.includes(p))) {
|
|
219
|
+
score += w.test;
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
score -= Math.min(pathDepth, 12);
|
|
223
|
+
|
|
224
|
+
return score;
|
|
225
|
+
};
|
|
226
|
+
|
|
227
|
+
const groupMatches = (matches, query, intent, indexHits, graphHits) => {
|
|
228
|
+
const groups = new Map();
|
|
229
|
+
|
|
230
|
+
for (const match of matches) {
|
|
231
|
+
if (!groups.has(match.file)) {
|
|
232
|
+
groups.set(match.file, []);
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
groups.get(match.file).push(match);
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const breakdown = { textMatch: 0, indexBoost: 0, graphBoost: 0 };
|
|
239
|
+
|
|
240
|
+
const sorted = [...groups.entries()]
|
|
241
|
+
.map(([file, fileMatches]) => {
|
|
242
|
+
let score = scoreGroup({ file, count: fileMatches.length, matches: fileMatches }, query, intent);
|
|
243
|
+
let boostSource = 'text';
|
|
244
|
+
if (indexHits?.has(file)) { score += 50; boostSource = 'index'; }
|
|
245
|
+
else if (graphHits?.has(file)) { score += 25; boostSource = 'graph'; }
|
|
246
|
+
return { file, count: fileMatches.length, score, matches: fileMatches, boostSource };
|
|
247
|
+
})
|
|
248
|
+
.sort((left, right) => right.score - left.score || right.count - left.count || left.file.localeCompare(right.file));
|
|
249
|
+
|
|
250
|
+
for (const g of sorted.slice(0, 10)) {
|
|
251
|
+
if (g.boostSource === 'index') breakdown.indexBoost++;
|
|
252
|
+
else if (g.boostSource === 'graph') breakdown.graphBoost++;
|
|
253
|
+
else breakdown.textMatch++;
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
return { groups: sorted, breakdown };
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
const buildCompactResult = (groups, totalMatches, query, root) => {
|
|
260
|
+
if (totalMatches <= 20) {
|
|
261
|
+
return groups
|
|
262
|
+
.flatMap((group) => group.matches)
|
|
263
|
+
.map(formatMatch)
|
|
264
|
+
.join('\n');
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
const lines = [
|
|
268
|
+
`query: ${query}`,
|
|
269
|
+
`root: ${root}`,
|
|
270
|
+
`total matches: ${totalMatches}`,
|
|
271
|
+
`matched files: ${groups.length}`,
|
|
272
|
+
'',
|
|
273
|
+
'# Top files',
|
|
274
|
+
];
|
|
275
|
+
|
|
276
|
+
for (const group of groups.slice(0, 10)) {
|
|
277
|
+
lines.push(`${group.count} match(es), score ${group.score} :: ${group.file}`);
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
lines.push('', '# Sample matches');
|
|
281
|
+
|
|
282
|
+
for (const group of groups.slice(0, 5)) {
|
|
283
|
+
for (const match of group.matches.slice(0, 3)) {
|
|
284
|
+
lines.push(formatMatch(match));
|
|
285
|
+
}
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
return lines.join('\n');
|
|
289
|
+
};
|
|
290
|
+
|
|
291
|
+
export const smartSearch = async ({ query, cwd = '.', intent, _testForceWalk = false }) => {
|
|
292
|
+
const root = resolveSafePath(cwd);
|
|
293
|
+
const rgMatches = _testForceWalk ? null : await searchWithRipgrep(root, query);
|
|
294
|
+
const usedFallback = rgMatches === null;
|
|
295
|
+
const engine = usedFallback ? 'walk' : 'rg';
|
|
296
|
+
|
|
297
|
+
let rawMatches;
|
|
298
|
+
let provenance;
|
|
299
|
+
|
|
300
|
+
if (usedFallback) {
|
|
301
|
+
const fallback = searchWithFallback(root, query);
|
|
302
|
+
rawMatches = fallback.matches;
|
|
303
|
+
const skippedTotal = fallback.skippedLarge + fallback.skippedBinary + fallback.skippedErrors + fallback.skippedDirs;
|
|
304
|
+
const warnings = ['search used filesystem walk instead of ripgrep'];
|
|
305
|
+
if (skippedTotal > 0) warnings.push(`${skippedTotal} items skipped (${fallback.skippedDirs} dirs, ${fallback.skippedLarge + fallback.skippedBinary + fallback.skippedErrors} files)`);
|
|
306
|
+
|
|
307
|
+
provenance = {
|
|
308
|
+
fallbackReason: 'rg unavailable or failed',
|
|
309
|
+
caseMode: fallback.caseSensitive ? 'sensitive' : 'insensitive',
|
|
310
|
+
partial: skippedTotal > 0,
|
|
311
|
+
skippedItemsTotal: skippedTotal,
|
|
312
|
+
skippedLargeFiles: fallback.skippedLarge,
|
|
313
|
+
skippedBinaryFiles: fallback.skippedBinary,
|
|
314
|
+
skippedReadErrors: fallback.skippedErrors,
|
|
315
|
+
skippedDirs: fallback.skippedDirs,
|
|
316
|
+
warnings,
|
|
317
|
+
};
|
|
318
|
+
} else {
|
|
319
|
+
rawMatches = rgMatches;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
rawMatches = rawMatches.filter((match) => !shouldIgnoreFile(match.file));
|
|
323
|
+
|
|
324
|
+
const seen = new Set();
|
|
325
|
+
const dedupedMatches = rawMatches.filter((match) => {
|
|
326
|
+
const key = `${match.file}:${match.lineNumber}:${match.content.trim()}`;
|
|
327
|
+
if (seen.has(key)) return false;
|
|
328
|
+
seen.add(key);
|
|
329
|
+
return true;
|
|
330
|
+
});
|
|
331
|
+
const validIntent = intent && VALID_INTENTS.has(intent) ? intent : undefined;
|
|
332
|
+
|
|
333
|
+
const indexRoot = projectRoot;
|
|
334
|
+
let indexHits = null;
|
|
335
|
+
let graphHits = null;
|
|
336
|
+
let indexFreshness = 'unavailable';
|
|
337
|
+
let loadedIndex = null;
|
|
338
|
+
try {
|
|
339
|
+
loadedIndex = loadIndex(indexRoot);
|
|
340
|
+
if (loadedIndex) {
|
|
341
|
+
indexFreshness = 'fresh';
|
|
342
|
+
const hits = queryIndex(loadedIndex, query);
|
|
343
|
+
if (hits.length > 0) {
|
|
344
|
+
indexHits = new Set(hits.map((h) => path.join(indexRoot, h.path)));
|
|
345
|
+
const related = new Set();
|
|
346
|
+
for (const h of hits) {
|
|
347
|
+
const rel = queryRelated(loadedIndex, h.path);
|
|
348
|
+
for (const p of [...rel.importedBy, ...rel.tests, ...rel.imports]) {
|
|
349
|
+
const full = path.join(indexRoot, p);
|
|
350
|
+
if (!indexHits.has(full)) related.add(full);
|
|
351
|
+
}
|
|
352
|
+
}
|
|
353
|
+
if (related.size > 0) graphHits = related;
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
} catch {
|
|
357
|
+
// index unavailable — continue without it
|
|
358
|
+
}
|
|
359
|
+
|
|
360
|
+
const { groups, breakdown } = groupMatches(dedupedMatches, query, validIntent, indexHits, graphHits);
|
|
361
|
+
|
|
362
|
+
if (loadedIndex && indexFreshness === 'fresh') {
|
|
363
|
+
const topRelPaths = groups.slice(0, 10).map((g) => path.relative(indexRoot, g.file).replace(/\\/g, '/'));
|
|
364
|
+
for (const rp of topRelPaths) {
|
|
365
|
+
const entry = loadedIndex.files?.[rp];
|
|
366
|
+
if (!entry) continue;
|
|
367
|
+
try {
|
|
368
|
+
const diskMtime = Math.floor(fs.statSync(path.join(indexRoot, rp)).mtimeMs);
|
|
369
|
+
if (diskMtime !== entry.mtime) { indexFreshness = 'stale'; break; }
|
|
370
|
+
} catch { /* file gone or unreadable */ }
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
|
|
374
|
+
const rawText = dedupedMatches.map(formatMatch).join('\n');
|
|
375
|
+
const compressedText = truncate(buildCompactResult(groups, dedupedMatches.length, query, root), 5000);
|
|
376
|
+
const metrics = buildMetrics({
|
|
377
|
+
tool: 'smart_search',
|
|
378
|
+
target: `${root} :: ${query}`,
|
|
379
|
+
rawText,
|
|
380
|
+
compressedText,
|
|
381
|
+
});
|
|
382
|
+
|
|
383
|
+
await persistMetrics(metrics);
|
|
384
|
+
|
|
385
|
+
let retrievalConfidence = 'high';
|
|
386
|
+
if (provenance) {
|
|
387
|
+
retrievalConfidence = provenance.skippedItemsTotal > 0 ? 'low' : 'medium';
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
const confidence = { level: retrievalConfidence, indexFreshness };
|
|
391
|
+
|
|
392
|
+
const result = {
|
|
393
|
+
query,
|
|
394
|
+
root,
|
|
395
|
+
engine,
|
|
396
|
+
retrievalConfidence,
|
|
397
|
+
indexFreshness,
|
|
398
|
+
sourceBreakdown: breakdown,
|
|
399
|
+
confidence,
|
|
400
|
+
...(validIntent ? { intent: validIntent } : {}),
|
|
401
|
+
...(indexHits ? { indexBoosted: indexHits.size } : {}),
|
|
402
|
+
totalMatches: dedupedMatches.length,
|
|
403
|
+
matchedFiles: groups.length,
|
|
404
|
+
topFiles: groups.slice(0, 10).map((group) => ({ file: group.file, count: group.count, score: group.score })),
|
|
405
|
+
matches: compressedText,
|
|
406
|
+
metrics,
|
|
407
|
+
};
|
|
408
|
+
|
|
409
|
+
if (provenance) result.provenance = provenance;
|
|
410
|
+
|
|
411
|
+
return result;
|
|
412
|
+
};
|
|
@@ -0,0 +1,213 @@
|
|
|
1
|
+
import { execFile as execFileCallback } from 'node:child_process';
|
|
2
|
+
import { promisify } from 'node:util';
|
|
3
|
+
import { rgPath } from '@vscode/ripgrep';
|
|
4
|
+
import { buildMetrics, persistMetrics } from '../metrics.js';
|
|
5
|
+
import { projectRoot } from '../utils/paths.js';
|
|
6
|
+
import { pickRelevantLines, truncate, uniqueLines } from '../utils/text.js';
|
|
7
|
+
|
|
8
|
+
const execFile = promisify(execFileCallback);
|
|
9
|
+
const blockedPattern = /[|&;<>`\n\r]/;
|
|
10
|
+
const allowedCommands = new Set(['pwd', 'ls', 'find', 'rg', 'git', 'npm', 'pnpm', 'yarn', 'bun']);
|
|
11
|
+
const allowedGitSubcommands = new Set(['status', 'diff', 'show', 'log', 'branch', 'rev-parse']);
|
|
12
|
+
const allowedPackageManagerSubcommands = new Set(['test', 'run', 'lint', 'build', 'typecheck', 'check']);
|
|
13
|
+
const safeRunScriptPattern = /^(test|lint|build|typecheck|check|smoke|verify)(:|$)/;
|
|
14
|
+
|
|
15
|
+
const tokenize = (command) => {
|
|
16
|
+
const tokens = [];
|
|
17
|
+
let current = '';
|
|
18
|
+
let quote = null;
|
|
19
|
+
let escape = false;
|
|
20
|
+
|
|
21
|
+
for (const char of command.trim()) {
|
|
22
|
+
if (escape) {
|
|
23
|
+
current += char;
|
|
24
|
+
escape = false;
|
|
25
|
+
continue;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
if (char === '\\') {
|
|
29
|
+
escape = true;
|
|
30
|
+
continue;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
if (quote) {
|
|
34
|
+
if (char === quote) {
|
|
35
|
+
quote = null;
|
|
36
|
+
} else {
|
|
37
|
+
current += char;
|
|
38
|
+
}
|
|
39
|
+
continue;
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
if (char === '"' || char === "'") {
|
|
43
|
+
quote = char;
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
if (/\s/.test(char)) {
|
|
48
|
+
if (current) {
|
|
49
|
+
tokens.push(current);
|
|
50
|
+
current = '';
|
|
51
|
+
}
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
current += char;
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
if (escape || quote) {
|
|
59
|
+
throw new Error('Unterminated escape or quote sequence');
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
if (current) {
|
|
63
|
+
tokens.push(current);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
return tokens;
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
const validateCommand = (command, tokens) => {
|
|
70
|
+
if (!command.trim()) {
|
|
71
|
+
return 'Command is empty';
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
if (blockedPattern.test(command) || command.includes('$(')) {
|
|
75
|
+
return 'Shell operators are not allowed';
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
if (tokens.length === 0) {
|
|
79
|
+
return 'Command is empty';
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
const [baseCommand, subcommand, thirdToken] = tokens;
|
|
83
|
+
|
|
84
|
+
if (!allowedCommands.has(baseCommand)) {
|
|
85
|
+
return `Command not allowed: ${baseCommand}`;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
if (baseCommand === 'git' && !allowedGitSubcommands.has(subcommand)) {
|
|
89
|
+
return `Git subcommand not allowed: ${subcommand ?? '(missing)'}`;
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
if (baseCommand === 'find') {
|
|
93
|
+
const dangerousArgs = ['-exec', '-execdir', '-delete', '-ok', '-okdir'];
|
|
94
|
+
const hasDangerous = tokens.some((t) => dangerousArgs.includes(t));
|
|
95
|
+
if (hasDangerous) {
|
|
96
|
+
return `find argument not allowed: ${tokens.find((t) => dangerousArgs.includes(t))}`;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
if (['npm', 'pnpm', 'yarn', 'bun'].includes(baseCommand)) {
|
|
101
|
+
if (!subcommand || !allowedPackageManagerSubcommands.has(subcommand)) {
|
|
102
|
+
return `Package manager subcommand not allowed: ${subcommand ?? '(missing)'}`;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
if (subcommand === 'run' && (!thirdToken || !safeRunScriptPattern.test(thirdToken))) {
|
|
106
|
+
return `Package manager script not allowed: ${thirdToken ?? '(missing)'}`;
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
return null;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
const buildBlockedResult = async (command, message) => {
|
|
114
|
+
const metrics = buildMetrics({
|
|
115
|
+
tool: 'smart_shell',
|
|
116
|
+
target: command,
|
|
117
|
+
rawText: command,
|
|
118
|
+
compressedText: message,
|
|
119
|
+
});
|
|
120
|
+
|
|
121
|
+
await persistMetrics(metrics);
|
|
122
|
+
|
|
123
|
+
return {
|
|
124
|
+
command,
|
|
125
|
+
exitCode: 126,
|
|
126
|
+
blocked: true,
|
|
127
|
+
output: message,
|
|
128
|
+
confidence: { blocked: true, timedOut: false },
|
|
129
|
+
metrics,
|
|
130
|
+
};
|
|
131
|
+
};
|
|
132
|
+
|
|
133
|
+
export const smartShell = async ({ command }) => {
|
|
134
|
+
let tokens;
|
|
135
|
+
|
|
136
|
+
try {
|
|
137
|
+
tokens = tokenize(command);
|
|
138
|
+
} catch (error) {
|
|
139
|
+
return await buildBlockedResult(command, error.message);
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
const validationError = validateCommand(command, tokens);
|
|
143
|
+
|
|
144
|
+
if (validationError) {
|
|
145
|
+
return await buildBlockedResult(command, validationError);
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const [file, ...args] = tokens;
|
|
149
|
+
|
|
150
|
+
if (file === 'find' && !args.includes('-maxdepth')) {
|
|
151
|
+
const findGlobalOptions = new Set(['-L', '-H', '-P', '-O0', '-O1', '-O2', '-O3', '-D']);
|
|
152
|
+
let insertAt = 0;
|
|
153
|
+
while (insertAt < args.length && findGlobalOptions.has(args[insertAt])) {
|
|
154
|
+
insertAt += 1;
|
|
155
|
+
if (args[insertAt - 1] === '-D' && insertAt < args.length) insertAt += 1;
|
|
156
|
+
}
|
|
157
|
+
while (insertAt < args.length && !args[insertAt].startsWith('-')) {
|
|
158
|
+
insertAt += 1;
|
|
159
|
+
}
|
|
160
|
+
args.splice(insertAt, 0, '-maxdepth', '8');
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
const resolvedFile = file === 'rg' ? rgPath : file;
|
|
164
|
+
const execution = await execFile(resolvedFile, args, {
|
|
165
|
+
cwd: projectRoot,
|
|
166
|
+
maxBuffer: 1024 * 1024 * 10,
|
|
167
|
+
timeout: 15000,
|
|
168
|
+
}).then(
|
|
169
|
+
({ stdout, stderr }) => ({ stdout, stderr, code: 0 }),
|
|
170
|
+
(error) => ({
|
|
171
|
+
stdout: error.stdout ?? '',
|
|
172
|
+
stderr: error.killed
|
|
173
|
+
? `Command timed out after 15s: ${command}`
|
|
174
|
+
: (error.stderr ?? error.message ?? ''),
|
|
175
|
+
code: Number.isInteger(error.code) ? error.code : 1,
|
|
176
|
+
timedOut: !!error.killed,
|
|
177
|
+
}),
|
|
178
|
+
);
|
|
179
|
+
|
|
180
|
+
const rawText = [execution.stdout, execution.stderr].filter(Boolean).join('\n');
|
|
181
|
+
const relevant = pickRelevantLines(rawText, [
|
|
182
|
+
'error',
|
|
183
|
+
'warning',
|
|
184
|
+
'failed',
|
|
185
|
+
'exception',
|
|
186
|
+
'maximum update depth',
|
|
187
|
+
'entity not found',
|
|
188
|
+
]);
|
|
189
|
+
const shouldPrioritizeRelevant = execution.code !== 0 || execution.timedOut;
|
|
190
|
+
const compressedSource = shouldPrioritizeRelevant && relevant ? relevant : rawText;
|
|
191
|
+
const compressedText = truncate(uniqueLines(compressedSource), 5000);
|
|
192
|
+
const metrics = buildMetrics({
|
|
193
|
+
tool: 'smart_shell',
|
|
194
|
+
target: command,
|
|
195
|
+
rawText,
|
|
196
|
+
compressedText,
|
|
197
|
+
});
|
|
198
|
+
|
|
199
|
+
await persistMetrics(metrics);
|
|
200
|
+
|
|
201
|
+
const result = {
|
|
202
|
+
command,
|
|
203
|
+
exitCode: execution.code,
|
|
204
|
+
blocked: false,
|
|
205
|
+
output: compressedText,
|
|
206
|
+
confidence: { blocked: false, timedOut: !!execution.timedOut },
|
|
207
|
+
metrics,
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
if (execution.timedOut) result.timedOut = true;
|
|
211
|
+
|
|
212
|
+
return result;
|
|
213
|
+
};
|
package/src/utils/fs.js
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { projectRoot } from './paths.js';
|
|
4
|
+
|
|
5
|
+
const assertInsideProject = (fullPath) => {
|
|
6
|
+
const relative = path.relative(projectRoot, fullPath);
|
|
7
|
+
|
|
8
|
+
if (relative.startsWith('..') || path.isAbsolute(relative)) {
|
|
9
|
+
throw new Error(`Path escapes project root: ${fullPath}`);
|
|
10
|
+
}
|
|
11
|
+
};
|
|
12
|
+
|
|
13
|
+
export const resolveSafePath = (inputPath = '.') => {
|
|
14
|
+
const fullPath = path.resolve(projectRoot, inputPath);
|
|
15
|
+
assertInsideProject(fullPath);
|
|
16
|
+
return fullPath;
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
const BINARY_CHECK_BYTES = 8192;
|
|
20
|
+
|
|
21
|
+
export const isBinaryBuffer = (buffer) => {
|
|
22
|
+
const length = Math.min(buffer.length, BINARY_CHECK_BYTES);
|
|
23
|
+
|
|
24
|
+
for (let i = 0; i < length; i++) {
|
|
25
|
+
const byte = buffer[i];
|
|
26
|
+
if (byte === 0) return true;
|
|
27
|
+
if (byte < 8 && byte !== 7) return true;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return false;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
export const isDockerfile = (filePath) => {
|
|
34
|
+
const baseName = path.basename(filePath).toLowerCase();
|
|
35
|
+
return baseName === 'dockerfile' || baseName.startsWith('dockerfile.');
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
export const readTextFile = (inputPath) => {
|
|
39
|
+
const fullPath = resolveSafePath(inputPath);
|
|
40
|
+
const raw = fs.readFileSync(fullPath);
|
|
41
|
+
|
|
42
|
+
if (isBinaryBuffer(raw)) {
|
|
43
|
+
throw new Error(`Binary file, cannot read as text: ${fullPath}`);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
return { fullPath, content: raw.toString('utf8') };
|
|
47
|
+
};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export { devctxRoot, projectRoot, projectRootSource, setProjectRoot } from './runtime-config.js';
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { fileURLToPath } from 'node:url';
|
|
3
|
+
|
|
4
|
+
const currentFilePath = fileURLToPath(import.meta.url);
|
|
5
|
+
const currentDir = path.dirname(currentFilePath);
|
|
6
|
+
|
|
7
|
+
const readArgValue = (name) => {
|
|
8
|
+
const index = process.argv.indexOf(name);
|
|
9
|
+
|
|
10
|
+
if (index === -1) {
|
|
11
|
+
return null;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
return process.argv[index + 1] ?? null;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
const defaultDevctxRoot = path.resolve(currentDir, '..', '..');
|
|
18
|
+
const defaultProjectRoot = path.resolve(defaultDevctxRoot, '..', '..');
|
|
19
|
+
const projectRootArg = readArgValue('--project-root');
|
|
20
|
+
const projectRootEnv = process.env.DEVCTX_PROJECT_ROOT ?? null;
|
|
21
|
+
const rawProjectRoot = projectRootArg ?? projectRootEnv ?? defaultProjectRoot;
|
|
22
|
+
|
|
23
|
+
export const devctxRoot = defaultDevctxRoot;
|
|
24
|
+
export let projectRoot = path.resolve(rawProjectRoot);
|
|
25
|
+
export const projectRootSource = projectRootArg ? 'argv' : projectRootEnv ? 'env' : 'default';
|
|
26
|
+
|
|
27
|
+
export const setProjectRoot = (newRoot) => {
|
|
28
|
+
projectRoot = path.resolve(newRoot);
|
|
29
|
+
};
|