skannr 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +22 -0
- package/README.md +109 -0
- package/dist/agent-cli.d.ts +7 -0
- package/dist/agent-cli.d.ts.map +1 -0
- package/dist/agent-cli.js +364 -0
- package/dist/agent-cli.js.map +1 -0
- package/dist/agent.d.ts +115 -0
- package/dist/agent.d.ts.map +1 -0
- package/dist/agent.js +340 -0
- package/dist/agent.js.map +1 -0
- package/dist/benchmark.d.ts +53 -0
- package/dist/benchmark.d.ts.map +1 -0
- package/dist/benchmark.js +307 -0
- package/dist/benchmark.js.map +1 -0
- package/dist/cache.d.ts +97 -0
- package/dist/cache.d.ts.map +1 -0
- package/dist/cache.js +284 -0
- package/dist/cache.js.map +1 -0
- package/dist/cli.d.ts +6 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +238 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +8 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +52 -0
- package/dist/config.js.map +1 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +176 -0
- package/dist/index.js.map +1 -0
- package/dist/languages/GenericAdapter.d.ts +10 -0
- package/dist/languages/GenericAdapter.d.ts.map +1 -0
- package/dist/languages/GenericAdapter.js +41 -0
- package/dist/languages/GenericAdapter.js.map +1 -0
- package/dist/languages/LanguageAdapter.d.ts +14 -0
- package/dist/languages/LanguageAdapter.d.ts.map +1 -0
- package/dist/languages/LanguageAdapter.js +3 -0
- package/dist/languages/LanguageAdapter.js.map +1 -0
- package/dist/languages/PythonAdapter.d.ts +10 -0
- package/dist/languages/PythonAdapter.d.ts.map +1 -0
- package/dist/languages/PythonAdapter.js +98 -0
- package/dist/languages/PythonAdapter.js.map +1 -0
- package/dist/languages/TypeScriptAdapter.d.ts +17 -0
- package/dist/languages/TypeScriptAdapter.d.ts.map +1 -0
- package/dist/languages/TypeScriptAdapter.js +321 -0
- package/dist/languages/TypeScriptAdapter.js.map +1 -0
- package/dist/languages/registry.d.ts +4 -0
- package/dist/languages/registry.d.ts.map +1 -0
- package/dist/languages/registry.js +86 -0
- package/dist/languages/registry.js.map +1 -0
- package/dist/mapper.d.ts +49 -0
- package/dist/mapper.d.ts.map +1 -0
- package/dist/mapper.js +386 -0
- package/dist/mapper.js.map +1 -0
- package/dist/ranker-enhanced.d.ts +37 -0
- package/dist/ranker-enhanced.d.ts.map +1 -0
- package/dist/ranker-enhanced.js +395 -0
- package/dist/ranker-enhanced.js.map +1 -0
- package/dist/ranker.d.ts +14 -0
- package/dist/ranker.d.ts.map +1 -0
- package/dist/ranker.js +105 -0
- package/dist/ranker.js.map +1 -0
- package/dist/rocket-chat-scope.d.ts +7 -0
- package/dist/rocket-chat-scope.d.ts.map +1 -0
- package/dist/rocket-chat-scope.js +95 -0
- package/dist/rocket-chat-scope.js.map +1 -0
- package/dist/scanner.d.ts +16 -0
- package/dist/scanner.d.ts.map +1 -0
- package/dist/scanner.js +228 -0
- package/dist/scanner.js.map +1 -0
- package/dist/skeletonizer.d.ts +5 -0
- package/dist/skeletonizer.d.ts.map +1 -0
- package/dist/skeletonizer.js +52 -0
- package/dist/skeletonizer.js.map +1 -0
- package/dist/tokenizer.d.ts +9 -0
- package/dist/tokenizer.d.ts.map +1 -0
- package/dist/tokenizer.js +21 -0
- package/dist/tokenizer.js.map +1 -0
- package/dist/types.d.ts +68 -0
- package/dist/types.d.ts.map +1 -0
- package/dist/types.js +6 -0
- package/dist/types.js.map +1 -0
- package/gemini-extension/GEMINI.md +26 -0
- package/gemini-extension/gemini-extension.json +12 -0
- package/gemini-extension/package.json +14 -0
- package/gemini-extension/src/server.ts +63 -0
- package/gemini-extension/tsconfig.json +14 -0
- package/jest.config.js +5 -0
- package/package.json +46 -0
- package/src/agent-cli.ts +383 -0
- package/src/agent.ts +344 -0
- package/src/benchmark.ts +389 -0
- package/src/cache.ts +317 -0
- package/src/cli.ts +223 -0
- package/src/config.ts +22 -0
- package/src/index.ts +215 -0
- package/src/languages/GenericAdapter.ts +44 -0
- package/src/languages/LanguageAdapter.ts +14 -0
- package/src/languages/PythonAdapter.ts +74 -0
- package/src/languages/TypeScriptAdapter.ts +338 -0
- package/src/languages/registry.ts +49 -0
- package/src/mapper.ts +448 -0
- package/src/ranker-enhanced.ts +460 -0
- package/src/ranker.ts +92 -0
- package/src/scanner.ts +201 -0
- package/src/skeletonizer.ts +16 -0
- package/src/tokenizer.ts +20 -0
- package/src/types.ts +71 -0
- package/tests/agent.tools.test.ts +81 -0
- package/tests/benchmark.test.ts +31 -0
- package/tests/fixtures/sample.py +17 -0
- package/tests/fixtures/sample.ts +13 -0
- package/tests/fixtures/src/api/routes.ts +1 -0
- package/tests/fixtures/src/auth/permission.ts +3 -0
- package/tests/ranker-enhanced.test.ts +68 -0
- package/tests/ranker.test.ts +75 -0
- package/tests/scanner.scope.test.ts +41 -0
- package/tests/setup-fixtures.js +29 -0
- package/tests/skeletonizer.test.ts +142 -0
- package/tsconfig.json +21 -0
- package/uca-landing/index.html +17 -0
- package/uca-landing/package.json +23 -0
- package/uca-landing/postcss.config.js +6 -0
- package/uca-landing/src/App.jsx +43 -0
- package/uca-landing/src/components/AgentMode.jsx +45 -0
- package/uca-landing/src/components/CliReference.jsx +49 -0
- package/uca-landing/src/components/Features.jsx +63 -0
- package/uca-landing/src/components/Footer.jsx +35 -0
- package/uca-landing/src/components/Hero.jsx +124 -0
- package/uca-landing/src/components/HowItWorks.jsx +60 -0
- package/uca-landing/src/components/Install.jsx +90 -0
- package/uca-landing/src/components/LanguageSupport.jsx +63 -0
- package/uca-landing/src/components/Navbar.jsx +86 -0
- package/uca-landing/src/components/Problem.jsx +51 -0
- package/uca-landing/src/components/Reveal.jsx +40 -0
- package/uca-landing/src/components/WorksWith.jsx +59 -0
- package/uca-landing/src/hooks/useScrollNav.js +13 -0
- package/uca-landing/src/hooks/useTypewriter.js +41 -0
- package/uca-landing/src/index.css +13 -0
- package/uca-landing/src/main.jsx +10 -0
- package/uca-landing/tailwind.config.js +68 -0
- package/uca-landing/vite.config.js +6 -0
package/src/benchmark.ts
ADDED
|
@@ -0,0 +1,389 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Benchmark script for code-analyzer
|
|
3
|
+
* Runs against a repository root to measure token reduction.
|
|
4
|
+
* Compares full-scan token cost vs skeletonized analysis across representative queries.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
import { analyzeProject } from './index';
|
|
8
|
+
import { scanTypeScriptFiles } from './scanner';
|
|
9
|
+
import { countTokens } from './tokenizer';
|
|
10
|
+
import * as path from 'path';
|
|
11
|
+
import * as fs from 'fs';
|
|
12
|
+
|
|
13
|
+
interface BenchmarkScenario {
|
|
14
|
+
moduleKey: string;
|
|
15
|
+
query: string;
|
|
16
|
+
limit: number;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
interface BenchmarkResult {
|
|
20
|
+
projectName: string;
|
|
21
|
+
projectRoot: string;
|
|
22
|
+
query: string;
|
|
23
|
+
// Baseline: what an LLM would consume reading ALL files
|
|
24
|
+
allFilesCount: number;
|
|
25
|
+
allFilesTotalTokens: number;
|
|
26
|
+
// Our tool output
|
|
27
|
+
analyzedFileCount: number;
|
|
28
|
+
analyzedOriginalTokens: number;
|
|
29
|
+
analyzedSkeletonTokens: number;
|
|
30
|
+
tokenReductionVsFullScan: number; // skeleton vs full-repo scan
|
|
31
|
+
tokenReductionVsTopN: number; // skeleton vs top-N original files
|
|
32
|
+
top1KeywordCoverage: number;
|
|
33
|
+
avgTopKKeywordCoverage: number;
|
|
34
|
+
pathIntentMatchRate: number;
|
|
35
|
+
directoryDiversity: number;
|
|
36
|
+
enhancedRanking: boolean;
|
|
37
|
+
executionTimeMs: number;
|
|
38
|
+
topFiles: {
|
|
39
|
+
path: string;
|
|
40
|
+
score: number;
|
|
41
|
+
originalTokens: number;
|
|
42
|
+
skeletonTokens: number;
|
|
43
|
+
reductionPct: number;
|
|
44
|
+
}[];
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
function extractQueryTerms(query: string): string[] {
|
|
48
|
+
return query
|
|
49
|
+
.toLowerCase()
|
|
50
|
+
.split(/[^a-z0-9]+/)
|
|
51
|
+
.filter(term => term.length > 2);
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function computeKeywordCoverage(pathValue: string, skeleton: string, terms: string[]): number {
|
|
55
|
+
if (terms.length === 0) return 0;
|
|
56
|
+
const haystack = `${pathValue} ${skeleton}`.toLowerCase();
|
|
57
|
+
const hits = terms.filter(term => haystack.includes(term)).length;
|
|
58
|
+
return hits / terms.length;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
function computeRankingQuality(
|
|
62
|
+
files: { path: string; skeleton: string }[],
|
|
63
|
+
query: string
|
|
64
|
+
): {
|
|
65
|
+
top1KeywordCoverage: number;
|
|
66
|
+
avgTopKKeywordCoverage: number;
|
|
67
|
+
pathIntentMatchRate: number;
|
|
68
|
+
directoryDiversity: number;
|
|
69
|
+
} {
|
|
70
|
+
if (files.length === 0) {
|
|
71
|
+
return {
|
|
72
|
+
top1KeywordCoverage: 0,
|
|
73
|
+
avgTopKKeywordCoverage: 0,
|
|
74
|
+
pathIntentMatchRate: 0,
|
|
75
|
+
directoryDiversity: 0,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const terms = extractQueryTerms(query);
|
|
80
|
+
const topK = files.slice(0, Math.min(5, files.length));
|
|
81
|
+
const coverages = topK.map(file => computeKeywordCoverage(file.path, file.skeleton, terms));
|
|
82
|
+
|
|
83
|
+
const top1KeywordCoverage = coverages[0] ?? 0;
|
|
84
|
+
const avgTopKKeywordCoverage = coverages.reduce((sum, value) => sum + value, 0) / topK.length;
|
|
85
|
+
|
|
86
|
+
const pathIntentMatches = topK.filter(file =>
|
|
87
|
+
terms.some(term => file.path.toLowerCase().includes(term))
|
|
88
|
+
).length;
|
|
89
|
+
const pathIntentMatchRate = pathIntentMatches / topK.length;
|
|
90
|
+
|
|
91
|
+
const uniqueDirs = new Set(
|
|
92
|
+
topK.map(file => path.dirname(file.path).toLowerCase())
|
|
93
|
+
);
|
|
94
|
+
const directoryDiversity = uniqueDirs.size / topK.length;
|
|
95
|
+
|
|
96
|
+
return {
|
|
97
|
+
top1KeywordCoverage,
|
|
98
|
+
avgTopKKeywordCoverage,
|
|
99
|
+
pathIntentMatchRate,
|
|
100
|
+
directoryDiversity,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
export const BENCHMARK_SCENARIOS: BenchmarkScenario[] = [
|
|
105
|
+
{ moduleKey: 'src', query: 'authentication flow and permission checks', limit: 10 },
|
|
106
|
+
{ moduleKey: 'lib', query: 'database connection management', limit: 10 },
|
|
107
|
+
{ moduleKey: 'packages', query: 'API endpoint and routing structure', limit: 10 },
|
|
108
|
+
];
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Quickly sum tokens across all .ts/.tsx files in a directory.
|
|
112
|
+
* Uses a capped sample to stay fast, then extrapolates.
|
|
113
|
+
*/
|
|
114
|
+
function sampleDirectoryTokens(
|
|
115
|
+
rootPath: string,
|
|
116
|
+
maxFiles = 50
|
|
117
|
+
): { fileCount: number; totalTokens: number; isSampled: boolean } {
|
|
118
|
+
const allFiles = scanTypeScriptFiles(rootPath);
|
|
119
|
+
const isSampled = allFiles.length > maxFiles;
|
|
120
|
+
const filesToRead = isSampled ? allFiles.slice(0, maxFiles) : allFiles;
|
|
121
|
+
|
|
122
|
+
let totalTokens = 0;
|
|
123
|
+
for (const f of filesToRead) {
|
|
124
|
+
try {
|
|
125
|
+
const content = fs.readFileSync(f, 'utf-8');
|
|
126
|
+
totalTokens += countTokens(content);
|
|
127
|
+
} catch {
|
|
128
|
+
// skip unreadable
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const estimatedTotal = isSampled
|
|
133
|
+
? Math.round((totalTokens / filesToRead.length) * allFiles.length)
|
|
134
|
+
: totalTokens;
|
|
135
|
+
|
|
136
|
+
return { fileCount: allFiles.length, totalTokens: estimatedTotal, isSampled };
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Run a single benchmark scenario
|
|
141
|
+
*/
|
|
142
|
+
async function benchmarkProject(
|
|
143
|
+
appRoot: string,
|
|
144
|
+
moduleKey: string,
|
|
145
|
+
projectName: string,
|
|
146
|
+
query: string,
|
|
147
|
+
options: { limit?: number; enhancedRanking?: boolean } = {}
|
|
148
|
+
): Promise<BenchmarkResult> {
|
|
149
|
+
const limit = options.limit ?? 10;
|
|
150
|
+
const moduleFullPath = fs.existsSync(path.join(appRoot, moduleKey))
|
|
151
|
+
? path.join(appRoot, moduleKey)
|
|
152
|
+
: appRoot;
|
|
153
|
+
|
|
154
|
+
// Baseline: estimate full-module token cost (what we are comparing against)
|
|
155
|
+
const baseline = sampleDirectoryTokens(moduleFullPath, 200);
|
|
156
|
+
|
|
157
|
+
const start = Date.now();
|
|
158
|
+
const result = await analyzeProject({
|
|
159
|
+
root: appRoot,
|
|
160
|
+
question: query,
|
|
161
|
+
limit,
|
|
162
|
+
enhancedRanking: options.enhancedRanking ?? true,
|
|
163
|
+
moduleKeys: [moduleKey],
|
|
164
|
+
generateMapping: false,
|
|
165
|
+
});
|
|
166
|
+
const executionTimeMs = Date.now() - start;
|
|
167
|
+
|
|
168
|
+
let analyzedOriginal = 0;
|
|
169
|
+
let analyzedSkeleton = 0;
|
|
170
|
+
|
|
171
|
+
const topFiles = result.files.map(f => {
|
|
172
|
+
analyzedOriginal += f.originalTokenCount;
|
|
173
|
+
analyzedSkeleton += f.skeletonTokenCount;
|
|
174
|
+
return {
|
|
175
|
+
path: f.path,
|
|
176
|
+
score: f.score,
|
|
177
|
+
originalTokens: f.originalTokenCount,
|
|
178
|
+
skeletonTokens: f.skeletonTokenCount,
|
|
179
|
+
reductionPct:
|
|
180
|
+
f.originalTokenCount > 0
|
|
181
|
+
? ((f.originalTokenCount - f.skeletonTokenCount) / f.originalTokenCount) * 100
|
|
182
|
+
: 0,
|
|
183
|
+
};
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
const tokenReductionVsFullScan =
|
|
187
|
+
baseline.totalTokens > 0
|
|
188
|
+
? ((baseline.totalTokens - analyzedSkeleton) / baseline.totalTokens) * 100
|
|
189
|
+
: 0;
|
|
190
|
+
|
|
191
|
+
const tokenReductionVsTopN =
|
|
192
|
+
analyzedOriginal > 0
|
|
193
|
+
? ((analyzedOriginal - analyzedSkeleton) / analyzedOriginal) * 100
|
|
194
|
+
: 0;
|
|
195
|
+
|
|
196
|
+
const rankingQuality = computeRankingQuality(
|
|
197
|
+
result.files.map(file => ({ path: file.path, skeleton: file.skeleton })),
|
|
198
|
+
query
|
|
199
|
+
);
|
|
200
|
+
|
|
201
|
+
return {
|
|
202
|
+
projectName,
|
|
203
|
+
projectRoot: moduleFullPath,
|
|
204
|
+
query,
|
|
205
|
+
allFilesCount: baseline.fileCount,
|
|
206
|
+
allFilesTotalTokens: baseline.totalTokens,
|
|
207
|
+
analyzedFileCount: result.files.length,
|
|
208
|
+
analyzedOriginalTokens: analyzedOriginal,
|
|
209
|
+
analyzedSkeletonTokens: analyzedSkeleton,
|
|
210
|
+
tokenReductionVsFullScan,
|
|
211
|
+
tokenReductionVsTopN,
|
|
212
|
+
top1KeywordCoverage: rankingQuality.top1KeywordCoverage,
|
|
213
|
+
avgTopKKeywordCoverage: rankingQuality.avgTopKKeywordCoverage,
|
|
214
|
+
pathIntentMatchRate: rankingQuality.pathIntentMatchRate,
|
|
215
|
+
directoryDiversity: rankingQuality.directoryDiversity,
|
|
216
|
+
enhancedRanking: options.enhancedRanking ?? true,
|
|
217
|
+
executionTimeMs,
|
|
218
|
+
topFiles,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Format results for console output
|
|
224
|
+
*/
|
|
225
|
+
function formatResults(results: BenchmarkResult[]): string {
|
|
226
|
+
const lines: string[] = [];
|
|
227
|
+
|
|
228
|
+
lines.push('═══════════════════════════════════════════════════════════════════════');
|
|
229
|
+
lines.push(' CODE ANALYZER BENCHMARK — Generic Repository');
|
|
230
|
+
lines.push('═══════════════════════════════════════════════════════════════════════\n');
|
|
231
|
+
|
|
232
|
+
for (const r of results) {
|
|
233
|
+
lines.push(`\n📊 ${r.projectName}`);
|
|
234
|
+
lines.push(` Query: "${r.query}"`);
|
|
235
|
+
lines.push('');
|
|
236
|
+
lines.push(` 🗂 Repository Baseline (naive full-scan):`);
|
|
237
|
+
lines.push(` Total .ts/.tsx files : ${r.allFilesCount.toLocaleString()}`);
|
|
238
|
+
lines.push(` Estimated tokens : ${r.allFilesTotalTokens.toLocaleString()}`);
|
|
239
|
+
lines.push('');
|
|
240
|
+
lines.push(` ⚙️ Our Tool (top ${r.analyzedFileCount} files, enhanced ranking: ${r.enhancedRanking ? '✓' : '✗'}):`);
|
|
241
|
+
lines.push(` Original tokens sent : ${r.analyzedOriginalTokens.toLocaleString()}`);
|
|
242
|
+
lines.push(` Skeleton tokens sent : ${r.analyzedSkeletonTokens.toLocaleString()}`);
|
|
243
|
+
lines.push(` Reduction vs full scan : ${r.tokenReductionVsFullScan.toFixed(1)}% ⬇️`);
|
|
244
|
+
lines.push(` Reduction vs top-N raw : ${r.tokenReductionVsTopN.toFixed(1)}% ⬇️`);
|
|
245
|
+
lines.push(` Top-1 keyword coverage : ${(r.top1KeywordCoverage * 100).toFixed(1)}%`);
|
|
246
|
+
lines.push(` Avg top-K coverage : ${(r.avgTopKKeywordCoverage * 100).toFixed(1)}%`);
|
|
247
|
+
lines.push(` Path intent match rate : ${(r.pathIntentMatchRate * 100).toFixed(1)}%`);
|
|
248
|
+
lines.push(` Directory diversity : ${(r.directoryDiversity * 100).toFixed(1)}%`);
|
|
249
|
+
lines.push(` Execution time : ${r.executionTimeMs}ms`);
|
|
250
|
+
lines.push('');
|
|
251
|
+
lines.push(` 📁 Top Ranked Files:`);
|
|
252
|
+
for (let i = 0; i < Math.min(5, r.topFiles.length); i++) {
|
|
253
|
+
const f = r.topFiles[i];
|
|
254
|
+
lines.push(
|
|
255
|
+
` ${i + 1}. ${f.path}`
|
|
256
|
+
);
|
|
257
|
+
lines.push(
|
|
258
|
+
` Score: ${f.score.toFixed(3)} | ${f.originalTokens} → ${f.skeletonTokens} tokens (${f.reductionPct.toFixed(1)}% ⬇️)`
|
|
259
|
+
);
|
|
260
|
+
}
|
|
261
|
+
lines.push('');
|
|
262
|
+
lines.push('───────────────────────────────────────────────────────────────────────');
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// Summary
|
|
266
|
+
const avgReductionFullScan =
|
|
267
|
+
results.reduce((s, r) => s + r.tokenReductionVsFullScan, 0) / results.length;
|
|
268
|
+
const avgReductionTopN =
|
|
269
|
+
results.reduce((s, r) => s + r.tokenReductionVsTopN, 0) / results.length;
|
|
270
|
+
const avgTop1Coverage =
|
|
271
|
+
results.reduce((s, r) => s + r.top1KeywordCoverage, 0) / results.length;
|
|
272
|
+
const avgTopKCoverage =
|
|
273
|
+
results.reduce((s, r) => s + r.avgTopKKeywordCoverage, 0) / results.length;
|
|
274
|
+
const avgPathIntentMatch =
|
|
275
|
+
results.reduce((s, r) => s + r.pathIntentMatchRate, 0) / results.length;
|
|
276
|
+
const avgTime =
|
|
277
|
+
results.reduce((s, r) => s + r.executionTimeMs, 0) / results.length;
|
|
278
|
+
|
|
279
|
+
lines.push('\n📊 SUMMARY');
|
|
280
|
+
lines.push('───────────────────────────────────────────────────────────────────────');
|
|
281
|
+
lines.push(` Benchmarks run : ${results.length}`);
|
|
282
|
+
lines.push(` Avg reduction vs full scan : ${avgReductionFullScan.toFixed(1)}%`);
|
|
283
|
+
lines.push(` Avg reduction vs top-N raw : ${avgReductionTopN.toFixed(1)}%`);
|
|
284
|
+
lines.push(` Avg top-1 coverage : ${(avgTop1Coverage * 100).toFixed(1)}%`);
|
|
285
|
+
lines.push(` Avg top-K coverage : ${(avgTopKCoverage * 100).toFixed(1)}%`);
|
|
286
|
+
lines.push(` Avg path intent match rate : ${(avgPathIntentMatch * 100).toFixed(1)}%`);
|
|
287
|
+
lines.push(` Avg execution time : ${avgTime.toFixed(0)}ms`);
|
|
288
|
+
lines.push('═══════════════════════════════════════════════════════════════════════\n');
|
|
289
|
+
|
|
290
|
+
return lines.join('\n');
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
/**
|
|
294
|
+
* Save benchmark output to JSON
|
|
295
|
+
*/
|
|
296
|
+
function saveResults(results: BenchmarkResult[], outputPath: string): void {
|
|
297
|
+
const out = {
|
|
298
|
+
generatedAt: new Date().toISOString(),
|
|
299
|
+
root: results[0]?.projectRoot ?? '',
|
|
300
|
+
summary: {
|
|
301
|
+
avgReductionVsFullScan:
|
|
302
|
+
results.reduce((s, r) => s + r.tokenReductionVsFullScan, 0) / results.length,
|
|
303
|
+
avgReductionVsTopN:
|
|
304
|
+
results.reduce((s, r) => s + r.tokenReductionVsTopN, 0) / results.length,
|
|
305
|
+
avgTop1KeywordCoverage:
|
|
306
|
+
results.reduce((s, r) => s + r.top1KeywordCoverage, 0) / results.length,
|
|
307
|
+
avgTopKKeywordCoverage:
|
|
308
|
+
results.reduce((s, r) => s + r.avgTopKKeywordCoverage, 0) / results.length,
|
|
309
|
+
avgPathIntentMatchRate:
|
|
310
|
+
results.reduce((s, r) => s + r.pathIntentMatchRate, 0) / results.length,
|
|
311
|
+
avgExecutionTimeMs:
|
|
312
|
+
results.reduce((s, r) => s + r.executionTimeMs, 0) / results.length,
|
|
313
|
+
},
|
|
314
|
+
benchmarks: results,
|
|
315
|
+
};
|
|
316
|
+
fs.writeFileSync(outputPath, JSON.stringify(out, null, 2), 'utf-8');
|
|
317
|
+
console.log(`\n✅ Results saved → ${outputPath}`);
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
/**
|
|
321
|
+
* Main entry point
|
|
322
|
+
*/
|
|
323
|
+
async function runBenchmarks(): Promise<void> {
|
|
324
|
+
const repoRoot = process.cwd();
|
|
325
|
+
|
|
326
|
+
// Verify repo exists
|
|
327
|
+
if (!fs.existsSync(repoRoot)) {
|
|
328
|
+
console.error(`❌ Repository not found at ${repoRoot}`);
|
|
329
|
+
process.exit(1);
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
const appRoot = repoRoot;
|
|
333
|
+
console.log('🚀 Code Analyzer — Generic Repository Benchmarks\n');
|
|
334
|
+
console.log(` Repo : ${repoRoot}`);
|
|
335
|
+
console.log(' Computing baseline (first 50 files sample)...');
|
|
336
|
+
const fullBaseline = sampleDirectoryTokens(appRoot, 50);
|
|
337
|
+
console.log(` Baseline → ${fullBaseline.fileCount} .ts files, ` +
|
|
338
|
+
`~${fullBaseline.totalTokens.toLocaleString()} tokens (${fullBaseline.isSampled ? 'extrapolated' : 'exact'})\n`);
|
|
339
|
+
|
|
340
|
+
const results: BenchmarkResult[] = [];
|
|
341
|
+
|
|
342
|
+
let i = 1;
|
|
343
|
+
for (const scenario of BENCHMARK_SCENARIOS) {
|
|
344
|
+
console.log(`▶ [${i}/${BENCHMARK_SCENARIOS.length}] ${scenario.query} (${scenario.moduleKey})`);
|
|
345
|
+
results.push(await benchmarkProject(
|
|
346
|
+
repoRoot,
|
|
347
|
+
scenario.moduleKey,
|
|
348
|
+
`Repo › ${scenario.moduleKey}`,
|
|
349
|
+
scenario.query,
|
|
350
|
+
{ limit: scenario.limit, enhancedRanking: true }
|
|
351
|
+
));
|
|
352
|
+
console.log(` ✓ ${results.at(-1)!.executionTimeMs}ms\n`);
|
|
353
|
+
i++;
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// ── Output ────────────────────────────────────────────────────────────────────
|
|
357
|
+
console.log(formatResults(results));
|
|
358
|
+
|
|
359
|
+
// Append full-app baseline note to the formatted output
|
|
360
|
+
console.log(`ℹ️ Full app/ baseline (naive full-scan):`);
|
|
361
|
+
console.log(` ${fullBaseline.fileCount} files → ~${fullBaseline.totalTokens.toLocaleString()} tokens`);
|
|
362
|
+
console.log(` vs our tool avg skeleton: ~${Math.round(results.reduce((s, r) => s + r.analyzedSkeletonTokens, 0) / results.length).toLocaleString()} tokens per query\n`);
|
|
363
|
+
|
|
364
|
+
const outputPath = path.join(__dirname, '..', 'benchmark-results.json');
|
|
365
|
+
// Attach baseline to JSON output
|
|
366
|
+
const extendedResults = {
|
|
367
|
+
fullAppBaseline: {
|
|
368
|
+
root: appRoot,
|
|
369
|
+
fileCount: fullBaseline.fileCount,
|
|
370
|
+
estimatedTotalTokens: fullBaseline.totalTokens,
|
|
371
|
+
isSampled: fullBaseline.isSampled,
|
|
372
|
+
},
|
|
373
|
+
benchmarks: results,
|
|
374
|
+
};
|
|
375
|
+
fs.writeFileSync(outputPath, JSON.stringify(extendedResults, null, 2), 'utf-8');
|
|
376
|
+
console.log(`✅ Saved → ${outputPath}`);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
// ── Run ───────────────────────────────────────────────────────────────────────
|
|
380
|
+
if (require.main === module) {
|
|
381
|
+
runBenchmarks()
|
|
382
|
+
.then(() => process.exit(0))
|
|
383
|
+
.catch(err => {
|
|
384
|
+
console.error('❌ Benchmark failed:', err);
|
|
385
|
+
process.exit(1);
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
export { benchmarkProject, formatResults, saveResults };
|
package/src/cache.ts
ADDED
|
@@ -0,0 +1,317 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Caching system for code analysis results
|
|
3
|
+
* Uses hybrid approach: in-memory + disk cache with MD5 file validation
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import * as fs from 'fs';
|
|
7
|
+
import * as path from 'path';
|
|
8
|
+
import * as crypto from 'crypto';
|
|
9
|
+
import { AnalysisResult } from './types';
|
|
10
|
+
|
|
11
|
+
export interface CacheConfig {
|
|
12
|
+
enabled: boolean;
|
|
13
|
+
inMemory: boolean;
|
|
14
|
+
onDisk: boolean;
|
|
15
|
+
cacheDir: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
export interface CacheEntry {
|
|
19
|
+
result: AnalysisResult;
|
|
20
|
+
fileHashes: Record<string, string>; // filename -> md5 hash
|
|
21
|
+
createdAt: number;
|
|
22
|
+
expiresAt: number;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Default cache configuration
|
|
27
|
+
*/
|
|
28
|
+
export const DEFAULT_CACHE_CONFIG: CacheConfig = {
|
|
29
|
+
enabled: true,
|
|
30
|
+
inMemory: true,
|
|
31
|
+
onDisk: true,
|
|
32
|
+
cacheDir: path.join(process.cwd(), '.code-analyzer-cache'),
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Cache manager for analysis results
|
|
37
|
+
*/
|
|
38
|
+
export class CacheManager {
|
|
39
|
+
private config: CacheConfig;
|
|
40
|
+
private memoryCache: Map<string, CacheEntry> = new Map();
|
|
41
|
+
private stats = {
|
|
42
|
+
hits: 0,
|
|
43
|
+
misses: 0,
|
|
44
|
+
invalidations: 0,
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
constructor(config: Partial<CacheConfig> = {}) {
|
|
48
|
+
this.config = { ...DEFAULT_CACHE_CONFIG, ...config };
|
|
49
|
+
|
|
50
|
+
if (this.config.onDisk && !fs.existsSync(this.config.cacheDir)) {
|
|
51
|
+
fs.mkdirSync(this.config.cacheDir, { recursive: true });
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* Generate cache key from analysis parameters
|
|
57
|
+
*/
|
|
58
|
+
private generateKey(
|
|
59
|
+
root: string,
|
|
60
|
+
question: string,
|
|
61
|
+
moduleKeys?: string[],
|
|
62
|
+
enhancedRanking?: boolean,
|
|
63
|
+
limit?: number
|
|
64
|
+
): string {
|
|
65
|
+
const input = `${root}|${question}|${(moduleKeys || []).join(',')}|${enhancedRanking}|${limit}`;
|
|
66
|
+
return crypto.createHash('md5').update(input).digest('hex');
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Calculate MD5 hash of file contents
|
|
71
|
+
*/
|
|
72
|
+
private hashFile(filePath: string): string | null {
|
|
73
|
+
try {
|
|
74
|
+
const content = fs.readFileSync(filePath, 'utf-8');
|
|
75
|
+
return crypto.createHash('md5').update(content).digest('hex');
|
|
76
|
+
} catch {
|
|
77
|
+
return null;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Hash all files in a directory (for change detection)
|
|
83
|
+
*/
|
|
84
|
+
private hashFiles(fileList: string[]): Record<string, string> {
|
|
85
|
+
const hashes: Record<string, string> = {};
|
|
86
|
+
|
|
87
|
+
for (const filePath of fileList) {
|
|
88
|
+
const hash = this.hashFile(filePath);
|
|
89
|
+
if (hash) {
|
|
90
|
+
hashes[filePath] = hash;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
return hashes;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Check if cached entry is still valid (files haven't changed)
|
|
99
|
+
*/
|
|
100
|
+
private isValidEntry(entry: CacheEntry, fileList: string[]): boolean {
|
|
101
|
+
// Check expiration (24 hours)
|
|
102
|
+
if (Date.now() > entry.expiresAt) {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Check if any files have changed
|
|
107
|
+
for (const filePath of fileList) {
|
|
108
|
+
const currentHash = this.hashFile(filePath);
|
|
109
|
+
const cachedHash = entry.fileHashes[filePath];
|
|
110
|
+
|
|
111
|
+
if (!currentHash || currentHash !== cachedHash) {
|
|
112
|
+
return false;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// Check if files were added/removed
|
|
117
|
+
if (Object.keys(entry.fileHashes).length !== fileList.length) {
|
|
118
|
+
return false;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
return true;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
/**
|
|
125
|
+
* Get cached result
|
|
126
|
+
*/
|
|
127
|
+
get(
|
|
128
|
+
root: string,
|
|
129
|
+
question: string,
|
|
130
|
+
analyzedFiles: string[],
|
|
131
|
+
moduleKeys?: string[],
|
|
132
|
+
enhancedRanking?: boolean,
|
|
133
|
+
limit?: number
|
|
134
|
+
): AnalysisResult | null {
|
|
135
|
+
if (!this.config.enabled) {
|
|
136
|
+
return null;
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
const key = this.generateKey(root, question, moduleKeys, enhancedRanking, limit);
|
|
140
|
+
|
|
141
|
+
// Try memory cache first
|
|
142
|
+
if (this.config.inMemory) {
|
|
143
|
+
const memEntry = this.memoryCache.get(key);
|
|
144
|
+
if (memEntry && this.isValidEntry(memEntry, analyzedFiles)) {
|
|
145
|
+
this.stats.hits++;
|
|
146
|
+
return memEntry.result;
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
// Try disk cache
|
|
151
|
+
if (this.config.onDisk) {
|
|
152
|
+
const diskEntry = this.loadFromDisk(key);
|
|
153
|
+
if (diskEntry && this.isValidEntry(diskEntry, analyzedFiles)) {
|
|
154
|
+
this.stats.hits++;
|
|
155
|
+
|
|
156
|
+
// Restore to memory cache
|
|
157
|
+
if (this.config.inMemory) {
|
|
158
|
+
this.memoryCache.set(key, diskEntry);
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
return diskEntry.result;
|
|
162
|
+
}
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
this.stats.misses++;
|
|
166
|
+
return null;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
/**
|
|
170
|
+
* Save result to cache
|
|
171
|
+
*/
|
|
172
|
+
set(
|
|
173
|
+
root: string,
|
|
174
|
+
question: string,
|
|
175
|
+
analyzedFiles: string[],
|
|
176
|
+
result: AnalysisResult,
|
|
177
|
+
moduleKeys?: string[],
|
|
178
|
+
enhancedRanking?: boolean,
|
|
179
|
+
limit?: number
|
|
180
|
+
): void {
|
|
181
|
+
if (!this.config.enabled) {
|
|
182
|
+
return;
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
const key = this.generateKey(root, question, moduleKeys, enhancedRanking, limit);
|
|
186
|
+
const fileHashes = this.hashFiles(analyzedFiles);
|
|
187
|
+
|
|
188
|
+
const entry: CacheEntry = {
|
|
189
|
+
result,
|
|
190
|
+
fileHashes,
|
|
191
|
+
createdAt: Date.now(),
|
|
192
|
+
expiresAt: Date.now() + 24 * 60 * 60 * 1000, // 24 hours
|
|
193
|
+
};
|
|
194
|
+
|
|
195
|
+
// Save to memory cache
|
|
196
|
+
if (this.config.inMemory) {
|
|
197
|
+
this.memoryCache.set(key, entry);
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// Save to disk cache
|
|
201
|
+
if (this.config.onDisk) {
|
|
202
|
+
this.saveToDisk(key, entry);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Load cache entry from disk
|
|
208
|
+
*/
|
|
209
|
+
private loadFromDisk(key: string): CacheEntry | null {
|
|
210
|
+
try {
|
|
211
|
+
const cacheFile = path.join(this.config.cacheDir, `${key}.json`);
|
|
212
|
+
if (!fs.existsSync(cacheFile)) {
|
|
213
|
+
return null;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
const content = fs.readFileSync(cacheFile, 'utf-8');
|
|
217
|
+
return JSON.parse(content) as CacheEntry;
|
|
218
|
+
} catch {
|
|
219
|
+
return null;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
/**
|
|
224
|
+
* Save cache entry to disk
|
|
225
|
+
*/
|
|
226
|
+
private saveToDisk(key: string, entry: CacheEntry): void {
|
|
227
|
+
try {
|
|
228
|
+
const cacheFile = path.join(this.config.cacheDir, `${key}.json`);
|
|
229
|
+
fs.writeFileSync(cacheFile, JSON.stringify(entry, null, 2), 'utf-8');
|
|
230
|
+
} catch (error) {
|
|
231
|
+
console.warn(`Failed to save cache to disk: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
232
|
+
}
|
|
233
|
+
}
|
|
234
|
+
|
|
235
|
+
/**
|
|
236
|
+
* Clear all caches
|
|
237
|
+
*/
|
|
238
|
+
clear(): void {
|
|
239
|
+
// Clear memory cache
|
|
240
|
+
this.memoryCache.clear();
|
|
241
|
+
|
|
242
|
+
// Clear disk cache
|
|
243
|
+
if (this.config.onDisk && fs.existsSync(this.config.cacheDir)) {
|
|
244
|
+
try {
|
|
245
|
+
fs.rmSync(this.config.cacheDir, { recursive: true, force: true });
|
|
246
|
+
fs.mkdirSync(this.config.cacheDir, { recursive: true });
|
|
247
|
+
} catch (error) {
|
|
248
|
+
console.warn(`Failed to clear disk cache: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
// Reset stats
|
|
253
|
+
this.stats = { hits: 0, misses: 0, invalidations: 0 };
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* Clear memory cache only
|
|
258
|
+
*/
|
|
259
|
+
clearMemory(): void {
|
|
260
|
+
this.memoryCache.clear();
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Get cache statistics
|
|
265
|
+
*/
|
|
266
|
+
getStats(): { hits: number; misses: number; invalidations: number; hitRate: number; cacheSize: number } {
|
|
267
|
+
const total = this.stats.hits + this.stats.misses;
|
|
268
|
+
const hitRate = total > 0 ? (this.stats.hits / total) * 100 : 0;
|
|
269
|
+
|
|
270
|
+
let cacheSize = 0;
|
|
271
|
+
if (this.config.onDisk && fs.existsSync(this.config.cacheDir)) {
|
|
272
|
+
const files = fs.readdirSync(this.config.cacheDir);
|
|
273
|
+
for (const file of files) {
|
|
274
|
+
const stat = fs.statSync(path.join(this.config.cacheDir, file));
|
|
275
|
+
cacheSize += stat.size;
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
return {
|
|
280
|
+
hits: this.stats.hits,
|
|
281
|
+
misses: this.stats.misses,
|
|
282
|
+
invalidations: this.stats.invalidations,
|
|
283
|
+
hitRate: Math.round(hitRate * 100) / 100,
|
|
284
|
+
cacheSize,
|
|
285
|
+
};
|
|
286
|
+
}
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Get cache directory path
|
|
290
|
+
*/
|
|
291
|
+
getCacheDir(): string {
|
|
292
|
+
return this.config.cacheDir;
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Global cache manager instance
|
|
298
|
+
*/
|
|
299
|
+
export let globalCacheManager: CacheManager | null = null;
|
|
300
|
+
|
|
301
|
+
/**
|
|
302
|
+
* Initialize global cache manager
|
|
303
|
+
*/
|
|
304
|
+
export function initializeCacheManager(config?: Partial<CacheConfig>): CacheManager {
|
|
305
|
+
globalCacheManager = new CacheManager(config);
|
|
306
|
+
return globalCacheManager;
|
|
307
|
+
}
|
|
308
|
+
|
|
309
|
+
/**
|
|
310
|
+
* Get or create global cache manager
|
|
311
|
+
*/
|
|
312
|
+
export function getCacheManager(): CacheManager {
|
|
313
|
+
if (!globalCacheManager) {
|
|
314
|
+
globalCacheManager = new CacheManager();
|
|
315
|
+
}
|
|
316
|
+
return globalCacheManager;
|
|
317
|
+
}
|