structx 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/benchmark/baseline.d.ts +10 -0
- package/dist/benchmark/baseline.d.ts.map +1 -0
- package/dist/benchmark/baseline.js +84 -0
- package/dist/benchmark/baseline.js.map +1 -0
- package/dist/benchmark/questions.d.ts +2 -0
- package/dist/benchmark/questions.d.ts.map +1 -0
- package/dist/benchmark/questions.js +14 -0
- package/dist/benchmark/questions.js.map +1 -0
- package/dist/benchmark/reporter.d.ts +8 -0
- package/dist/benchmark/reporter.d.ts.map +1 -0
- package/dist/benchmark/reporter.js +120 -0
- package/dist/benchmark/reporter.js.map +1 -0
- package/dist/benchmark/runner.d.ts +24 -0
- package/dist/benchmark/runner.d.ts.map +1 -0
- package/dist/benchmark/runner.js +110 -0
- package/dist/benchmark/runner.js.map +1 -0
- package/dist/cli.d.ts +3 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +753 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +15 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +83 -0
- package/dist/config.js.map +1 -0
- package/dist/db/connection.d.ts +5 -0
- package/dist/db/connection.d.ts.map +1 -0
- package/dist/db/connection.js +89 -0
- package/dist/db/connection.js.map +1 -0
- package/dist/db/queries.d.ts +122 -0
- package/dist/db/queries.d.ts.map +1 -0
- package/dist/db/queries.js +191 -0
- package/dist/db/queries.js.map +1 -0
- package/dist/db/schema.sql +85 -0
- package/dist/ingest/differ.d.ts +13 -0
- package/dist/ingest/differ.d.ts.map +1 -0
- package/dist/ingest/differ.js +63 -0
- package/dist/ingest/differ.js.map +1 -0
- package/dist/ingest/parser.d.ts +15 -0
- package/dist/ingest/parser.d.ts.map +1 -0
- package/dist/ingest/parser.js +154 -0
- package/dist/ingest/parser.js.map +1 -0
- package/dist/ingest/relationships.d.ts +8 -0
- package/dist/ingest/relationships.d.ts.map +1 -0
- package/dist/ingest/relationships.js +93 -0
- package/dist/ingest/relationships.js.map +1 -0
- package/dist/ingest/scanner.d.ts +2 -0
- package/dist/ingest/scanner.d.ts.map +1 -0
- package/dist/ingest/scanner.js +67 -0
- package/dist/ingest/scanner.js.map +1 -0
- package/dist/instructions/claude.md +41 -0
- package/dist/instructions/copilot.md +39 -0
- package/dist/instructions/cursor.md +39 -0
- package/dist/instructions/generic.md +41 -0
- package/dist/query/answerer.d.ts +9 -0
- package/dist/query/answerer.d.ts.map +1 -0
- package/dist/query/answerer.js +46 -0
- package/dist/query/answerer.js.map +1 -0
- package/dist/query/classifier.d.ts +10 -0
- package/dist/query/classifier.d.ts.map +1 -0
- package/dist/query/classifier.js +60 -0
- package/dist/query/classifier.js.map +1 -0
- package/dist/query/context-builder.d.ts +3 -0
- package/dist/query/context-builder.d.ts.map +1 -0
- package/dist/query/context-builder.js +104 -0
- package/dist/query/context-builder.js.map +1 -0
- package/dist/query/retriever.d.ts +23 -0
- package/dist/query/retriever.d.ts.map +1 -0
- package/dist/query/retriever.js +142 -0
- package/dist/query/retriever.js.map +1 -0
- package/dist/semantic/analyzer.d.ts +15 -0
- package/dist/semantic/analyzer.d.ts.map +1 -0
- package/dist/semantic/analyzer.js +179 -0
- package/dist/semantic/analyzer.js.map +1 -0
- package/dist/semantic/cost.d.ts +11 -0
- package/dist/semantic/cost.d.ts.map +1 -0
- package/dist/semantic/cost.js +31 -0
- package/dist/semantic/cost.js.map +1 -0
- package/dist/semantic/prompt.d.ts +11 -0
- package/dist/semantic/prompt.d.ts.map +1 -0
- package/dist/semantic/prompt.js +71 -0
- package/dist/semantic/prompt.js.map +1 -0
- package/dist/semantic/validator.d.ts +15 -0
- package/dist/semantic/validator.d.ts.map +1 -0
- package/dist/semantic/validator.js +95 -0
- package/dist/semantic/validator.js.map +1 -0
- package/dist/utils/logger.d.ts +9 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +48 -0
- package/dist/utils/logger.js.map +1 -0
- package/dist/utils/tokens.d.ts +3 -0
- package/dist/utils/tokens.d.ts.map +1 -0
- package/dist/utils/tokens.js +21 -0
- package/dist/utils/tokens.js.map +1 -0
- package/package.json +39 -0
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface BaselineResult {
|
|
2
|
+
answer: string;
|
|
3
|
+
inputTokens: number;
|
|
4
|
+
outputTokens: number;
|
|
5
|
+
cost: number;
|
|
6
|
+
responseTimeMs: number;
|
|
7
|
+
filesAccessed: number;
|
|
8
|
+
}
|
|
9
|
+
export declare function runBaseline(question: string, repoPath: string, model: string, apiKey: string): Promise<BaselineResult>;
|
|
10
|
+
//# sourceMappingURL=baseline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"baseline.d.ts","sourceRoot":"","sources":["../../src/benchmark/baseline.ts"],"names":[],"mappings":"AAMA,MAAM,WAAW,cAAc;IAC7B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,EAAE,MAAM,CAAC;IACpB,YAAY,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,cAAc,EAAE,MAAM,CAAC;IACvB,aAAa,EAAE,MAAM,CAAC;CACvB;AAED,wBAAsB,WAAW,CAC/B,QAAQ,EAAE,MAAM,EAChB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,EACb,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,cAAc,CAAC,CA+CzB"}
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
36
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
37
|
+
};
|
|
38
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
39
|
+
exports.runBaseline = runBaseline;
|
|
40
|
+
const sdk_1 = __importDefault(require("@anthropic-ai/sdk"));
|
|
41
|
+
const fs = __importStar(require("fs"));
|
|
42
|
+
const path = __importStar(require("path"));
|
|
43
|
+
const scanner_1 = require("../ingest/scanner");
|
|
44
|
+
const tokens_1 = require("../utils/tokens");
|
|
45
|
+
async function runBaseline(question, repoPath, model, apiKey) {
|
|
46
|
+
const client = new sdk_1.default({ apiKey });
|
|
47
|
+
// Read all TypeScript files
|
|
48
|
+
const files = (0, scanner_1.scanDirectory)(repoPath);
|
|
49
|
+
let codeContext = '';
|
|
50
|
+
let filesAccessed = 0;
|
|
51
|
+
for (const filePath of files) {
|
|
52
|
+
const relativePath = path.relative(repoPath, filePath);
|
|
53
|
+
const content = fs.readFileSync(filePath, 'utf-8');
|
|
54
|
+
codeContext += `\n--- ${relativePath} ---\n${content}\n`;
|
|
55
|
+
filesAccessed++;
|
|
56
|
+
}
|
|
57
|
+
const prompt = `You are a code analysis assistant. Below is the complete source code of a TypeScript project. Answer the developer's question based on this code.
|
|
58
|
+
|
|
59
|
+
${codeContext}
|
|
60
|
+
|
|
61
|
+
Question: ${question}`;
|
|
62
|
+
const startTime = Date.now();
|
|
63
|
+
const response = await client.messages.create({
|
|
64
|
+
model,
|
|
65
|
+
max_tokens: 1024,
|
|
66
|
+
messages: [{ role: 'user', content: prompt }],
|
|
67
|
+
});
|
|
68
|
+
const responseTimeMs = Date.now() - startTime;
|
|
69
|
+
const answer = response.content
|
|
70
|
+
.filter(block => block.type === 'text')
|
|
71
|
+
.map(block => block.text)
|
|
72
|
+
.join('');
|
|
73
|
+
const inputTokens = response.usage?.input_tokens ?? 0;
|
|
74
|
+
const outputTokens = response.usage?.output_tokens ?? 0;
|
|
75
|
+
return {
|
|
76
|
+
answer,
|
|
77
|
+
inputTokens,
|
|
78
|
+
outputTokens,
|
|
79
|
+
cost: (0, tokens_1.estimateCost)(model, inputTokens, outputTokens),
|
|
80
|
+
responseTimeMs,
|
|
81
|
+
filesAccessed,
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
//# sourceMappingURL=baseline.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"baseline.js","sourceRoot":"","sources":["../../src/benchmark/baseline.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAeA,kCAoDC;AAnED,4DAA0C;AAC1C,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAkD;AAClD,4CAA+C;AAWxC,KAAK,UAAU,WAAW,CAC/B,QAAgB,EAChB,QAAgB,EAChB,KAAa,EACb,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,aAAS,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC;IAEzC,4BAA4B;IAC5B,MAAM,KAAK,GAAG,IAAA,uBAAa,EAAC,QAAQ,CAAC,CAAC;IACtC,IAAI,WAAW,GAAG,EAAE,CAAC;IACrB,IAAI,aAAa,GAAG,CAAC,CAAC;IAEtB,KAAK,MAAM,QAAQ,IAAI,KAAK,EAAE,CAAC;QAC7B,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC;QACvD,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACnD,WAAW,IAAI,SAAS,YAAY,SAAS,OAAO,IAAI,CAAC;QACzD,aAAa,EAAE,CAAC;IAClB,CAAC;IAED,MAAM,MAAM,GAAG;;EAEf,WAAW;;YAED,QAAQ,EAAE,CAAC;IAErB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAC5C,KAAK;QACL,UAAU,EAAE,IAAI;QAChB,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC;KAC9C,CAAC,CAAC;IAEH,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;IAE9C,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO;SAC5B,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,MAAM,CAAC;SACtC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAE,KAAa,CAAC,IAAI,CAAC;SACjC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEZ,MAAM,WAAW,GAAG,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,CAAC;IACtD,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,EAAE,aAAa,IAAI,CAAC,CAAC;IAExD,OAAO;QACL,MAAM;QACN,WAAW;QACX,YAAY;QACZ,IAAI,EAAE,IAAA,qBAAY,EAAC,KAAK,EAAE,WAAW,EAAE,YAAY,CAAC;QACpD,cAAc;QACd,aAAa;KACd,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"questions.d.ts","sourceRoot":"","sources":["../../src/benchmark/questions.ts"],"names":[],"mappings":"AAAA,eAAO,MAAM,mBAAmB,UAS/B,CAAC"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BENCHMARK_QUESTIONS = void 0;
|
|
4
|
+
exports.BENCHMARK_QUESTIONS = [
|
|
5
|
+
'What functions handle user authentication?',
|
|
6
|
+
'What does the login function call?',
|
|
7
|
+
'Show me all functions that use Redis',
|
|
8
|
+
'What calls the validatePassword function?',
|
|
9
|
+
'How is session management implemented?',
|
|
10
|
+
'What database operations modify user data?',
|
|
11
|
+
'Which functions have side effects?',
|
|
12
|
+
'Find all async functions in the authentication domain',
|
|
13
|
+
];
|
|
14
|
+
//# sourceMappingURL=questions.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"questions.js","sourceRoot":"","sources":["../../src/benchmark/questions.ts"],"names":[],"mappings":";;;AAAa,QAAA,mBAAmB,GAAG;IACjC,4CAA4C;IAC5C,oCAAoC;IACpC,sCAAsC;IACtC,2CAA2C;IAC3C,wCAAwC;IACxC,4CAA4C;IAC5C,oCAAoC;IACpC,uDAAuD;CACxD,CAAC"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import type { BenchmarkRunResult } from './runner';
|
|
2
|
+
export declare function generateMarkdownReport(results: BenchmarkRunResult[]): string;
|
|
3
|
+
export declare function generateCsvReport(results: BenchmarkRunResult[]): string;
|
|
4
|
+
export declare function saveReport(structxDir: string, markdownContent: string, csvContent: string): {
|
|
5
|
+
markdownPath: string;
|
|
6
|
+
csvPath: string;
|
|
7
|
+
};
|
|
8
|
+
//# sourceMappingURL=reporter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.d.ts","sourceRoot":"","sources":["../../src/benchmark/reporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAInD,wBAAgB,sBAAsB,CAAC,OAAO,EAAE,kBAAkB,EAAE,GAAG,MAAM,CAuD5E;AAED,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,kBAAkB,EAAE,GAAG,MAAM,CAevE;AAED,wBAAgB,UAAU,CACxB,UAAU,EAAE,MAAM,EAClB,eAAe,EAAE,MAAM,EACvB,UAAU,EAAE,MAAM,GACjB;IAAE,YAAY,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,CAAA;CAAE,CAc3C"}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.generateMarkdownReport = generateMarkdownReport;
|
|
37
|
+
exports.generateCsvReport = generateCsvReport;
|
|
38
|
+
exports.saveReport = saveReport;
|
|
39
|
+
const fs = __importStar(require("fs"));
|
|
40
|
+
const path = __importStar(require("path"));
|
|
41
|
+
function generateMarkdownReport(results) {
|
|
42
|
+
const lines = [];
|
|
43
|
+
lines.push('# StructX Benchmark Report');
|
|
44
|
+
lines.push('');
|
|
45
|
+
lines.push(`Generated: ${new Date().toISOString()}`);
|
|
46
|
+
lines.push('');
|
|
47
|
+
// Summary table
|
|
48
|
+
lines.push('## Results');
|
|
49
|
+
lines.push('');
|
|
50
|
+
lines.push('| # | Question | Mode | Input Tokens | Output Tokens | Cost | Time (ms) | Context |');
|
|
51
|
+
lines.push('|---|----------|------|-------------|---------------|------|-----------|---------|');
|
|
52
|
+
for (let i = 0; i < results.length; i++) {
|
|
53
|
+
const r = results[i];
|
|
54
|
+
const shortQ = r.question.length > 40 ? r.question.slice(0, 40) + '...' : r.question;
|
|
55
|
+
if (r.structx) {
|
|
56
|
+
lines.push(`| ${i + 1} | ${shortQ} | StructX | ${r.structx.inputTokens} | ${r.structx.outputTokens} | $${r.structx.cost.toFixed(4)} | ${r.structx.responseTimeMs} | ${r.structx.functionsRetrieved} functions |`);
|
|
57
|
+
}
|
|
58
|
+
if (r.traditional) {
|
|
59
|
+
lines.push(`| ${i + 1} | ${shortQ} | Traditional | ${r.traditional.inputTokens} | ${r.traditional.outputTokens} | $${r.traditional.cost.toFixed(4)} | ${r.traditional.responseTimeMs} | ${r.traditional.filesAccessed} files |`);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
// Aggregate statistics
|
|
63
|
+
lines.push('');
|
|
64
|
+
lines.push('## Summary Statistics');
|
|
65
|
+
lines.push('');
|
|
66
|
+
const structxRuns = results.filter(r => r.structx).map(r => r.structx);
|
|
67
|
+
const traditionalRuns = results.filter(r => r.traditional).map(r => r.traditional);
|
|
68
|
+
if (structxRuns.length > 0 && traditionalRuns.length > 0) {
|
|
69
|
+
const avgStructxTokens = avg(structxRuns.map(r => r.inputTokens + r.outputTokens));
|
|
70
|
+
const avgTraditionalTokens = avg(traditionalRuns.map(r => r.inputTokens + r.outputTokens));
|
|
71
|
+
const tokenReduction = ((avgTraditionalTokens - avgStructxTokens) / avgTraditionalTokens * 100);
|
|
72
|
+
const avgStructxCost = avg(structxRuns.map(r => r.cost));
|
|
73
|
+
const avgTraditionalCost = avg(traditionalRuns.map(r => r.cost));
|
|
74
|
+
const costReduction = ((avgTraditionalCost - avgStructxCost) / avgTraditionalCost * 100);
|
|
75
|
+
const avgStructxTime = avg(structxRuns.map(r => r.responseTimeMs));
|
|
76
|
+
const avgTraditionalTime = avg(traditionalRuns.map(r => r.responseTimeMs));
|
|
77
|
+
lines.push('| Metric | StructX | Traditional | Improvement |');
|
|
78
|
+
lines.push('|--------|---------|-------------|-------------|');
|
|
79
|
+
lines.push(`| Avg Tokens | ${avgStructxTokens.toFixed(0)} | ${avgTraditionalTokens.toFixed(0)} | ${tokenReduction.toFixed(1)}% reduction |`);
|
|
80
|
+
lines.push(`| Avg Cost | $${avgStructxCost.toFixed(4)} | $${avgTraditionalCost.toFixed(4)} | ${costReduction.toFixed(1)}% reduction |`);
|
|
81
|
+
lines.push(`| Avg Time | ${avgStructxTime.toFixed(0)}ms | ${avgTraditionalTime.toFixed(0)}ms | ${((avgTraditionalTime - avgStructxTime) / avgTraditionalTime * 100).toFixed(1)}% faster |`);
|
|
82
|
+
lines.push(`| Total Cost | $${sum(structxRuns.map(r => r.cost)).toFixed(4)} | $${sum(traditionalRuns.map(r => r.cost)).toFixed(4)} | |`);
|
|
83
|
+
}
|
|
84
|
+
return lines.join('\n');
|
|
85
|
+
}
|
|
86
|
+
function generateCsvReport(results) {
|
|
87
|
+
const lines = [];
|
|
88
|
+
lines.push('question,mode,input_tokens,output_tokens,total_tokens,cost_usd,response_time_ms,context_size');
|
|
89
|
+
for (const r of results) {
|
|
90
|
+
const q = `"${r.question.replace(/"/g, '""')}"`;
|
|
91
|
+
if (r.structx) {
|
|
92
|
+
lines.push(`${q},structx,${r.structx.inputTokens},${r.structx.outputTokens},${r.structx.inputTokens + r.structx.outputTokens},${r.structx.cost.toFixed(6)},${r.structx.responseTimeMs},${r.structx.functionsRetrieved}`);
|
|
93
|
+
}
|
|
94
|
+
if (r.traditional) {
|
|
95
|
+
lines.push(`${q},traditional,${r.traditional.inputTokens},${r.traditional.outputTokens},${r.traditional.inputTokens + r.traditional.outputTokens},${r.traditional.cost.toFixed(6)},${r.traditional.responseTimeMs},${r.traditional.filesAccessed}`);
|
|
96
|
+
}
|
|
97
|
+
}
|
|
98
|
+
return lines.join('\n');
|
|
99
|
+
}
|
|
100
|
+
function saveReport(structxDir, markdownContent, csvContent) {
|
|
101
|
+
const reportsDir = path.join(structxDir, 'reports');
|
|
102
|
+
if (!fs.existsSync(reportsDir)) {
|
|
103
|
+
fs.mkdirSync(reportsDir, { recursive: true });
|
|
104
|
+
}
|
|
105
|
+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-').slice(0, 19);
|
|
106
|
+
const markdownPath = path.join(reportsDir, `benchmark-${timestamp}.md`);
|
|
107
|
+
const csvPath = path.join(reportsDir, `benchmark-${timestamp}.csv`);
|
|
108
|
+
fs.writeFileSync(markdownPath, markdownContent, 'utf-8');
|
|
109
|
+
fs.writeFileSync(csvPath, csvContent, 'utf-8');
|
|
110
|
+
return { markdownPath, csvPath };
|
|
111
|
+
}
|
|
112
|
+
function avg(nums) {
|
|
113
|
+
if (nums.length === 0)
|
|
114
|
+
return 0;
|
|
115
|
+
return nums.reduce((a, b) => a + b, 0) / nums.length;
|
|
116
|
+
}
|
|
117
|
+
function sum(nums) {
|
|
118
|
+
return nums.reduce((a, b) => a + b, 0);
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=reporter.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"reporter.js","sourceRoot":"","sources":["../../src/benchmark/reporter.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIA,wDAuDC;AAED,8CAeC;AAED,gCAkBC;AA/FD,uCAAyB;AACzB,2CAA6B;AAE7B,SAAgB,sBAAsB,CAAC,OAA6B;IAClE,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,KAAK,CAAC,IAAI,CAAC,4BAA4B,CAAC,CAAC;IACzC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,cAAc,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;IACrD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,qFAAqF,CAAC,CAAC;IAClG,KAAK,CAAC,IAAI,CAAC,oFAAoF,CAAC,CAAC;IAEjG,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,MAAM,MAAM,GAAG,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC;QAErF,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,MAAM,gBAAgB,CAAC,CAAC,OAAO,CAAC,WAAW,MAAM,CAAC,CAAC,OAAO,CAAC,YAAY,OAAO,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,cAAc,MAAM,CAAC,CAAC,OAAO,CAAC,kBAAkB,cAAc,CAAC,CAAC;QACpN,CAAC;QACD,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,MAAM,oBAAoB,CAAC,CAAC,WAAW,CAAC,WAAW,MAAM,CAAC,CAAC,WAAW,CAAC,YAAY,OAAO,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,WAAW,CAAC,cAAc,MAAM,CAAC,CAAC,WAAW,CAAC,aAAa,UAAU,CAAC,CAAC;QACnO,CAAC;IACH,CAAC;IAED,uBAAuB;IACvB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,uBAAuB,CAAC,CAAC;IACpC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAQ,CAAC,CAAC;IACxE,MAAM,eAAe,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAY,CAAC,CAAC;IAEpF,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,IAAI,eAAe,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzD,MAAM,gBAAgB,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;QACnF,MAAM,oBAAoB,GAAG,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;QAC3F,MAAM,cAAc,GAAG,CAAC,CAAC,oBAAoB,GAAG,gBAAgB,CAAC,GAAG,oBAAoB,GAAG,GAAG,CAAC,CAAC;QAEhG,MAAM,cAAc,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACzD,MAAM,kBAAkB,GAAG,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QACjE,MAAM,aAAa,GAAG,CAAC,CAAC,kBAAkB,GAAG,cAAc,CAAC,GAAG,kBAAkB,GAAG,GAAG,CAAC,CAAC;QAEzF,MAAM,cAAc,GAAG,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC;QACnE,MAAM,kBAAkB,GAAG,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC,CAAC;QAE3E,KAAK,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QAC/D,KAAK,CAAC,IAAI,CAAC,kDAAkD,CAAC,CAAC;QAC/D,KAAK,CAAC,IAAI,CAAC,kBAAkB,gBAAgB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,oBAAoB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QAC7I,KAAK,CAAC,IAAI,CAAC,iBAAiB,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC;QACxI,KAAK,CAAC,IAAI,CAAC,gBAAgB,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,kBAAkB,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,kBAAkB,GAAG,cAAc,CAAC,GAAG,kBAAkB,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QAC5L,KAAK,CAAC,IAAI,CAAC,mBAAmB,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,OAAO,GAAG,CAAC,eAAe,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;IAC3I,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAgB,iBAAiB,CAAC,OAA6B;IAC7D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,8FAA8F,CAAC,CAAC;IAE3G,KAAK,MAAM,CAAC,IAAI,OAAO,EAAE,CAAC;QACxB,MAAM,CAAC,GAAG,IAAI,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;QAChD,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;YACd,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,OAAO,CAAC,WAAW,IAAI,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,OAAO,CAAC,WAAW,GAAG,CAAC,CAAC,OAAO,CAAC,YAAY,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,cAAc,IAAI,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,CAAC,CAAC;QAC3N,CAAC;QACD,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,WAAW,CAAC,WAAW,IAAI,CAAC,CAAC,WAAW,CAAC,YAAY,IAAI,CAAC,CAAC,WAAW,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC,YAAY,IAAI,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,WAAW,CAAC,cAAc,IAAI,CAAC,CAAC,WAAW,CAAC,aAAa,EAAE,CAAC,CAAC;QACtP,CAAC;IACH,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,SAAgB,UAAU,CACxB,UAAkB,EAClB,eAAuB,EACvB,UAAkB;IAElB,MAAM,UAAU,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACpD,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,EAAE,CAAC,SAAS,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAChD,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAC9E,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,aAAa,SAAS,KAAK,CAAC,CAAC;IACxE,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,aAAa,SAAS,MAAM,CAAC,CAAC;IAEpE,EAAE,CAAC,aAAa,CAAC,YAAY,EAAE,eAAe,EAAE,OAAO,CAAC,CAAC;IACzD,EAAE,CAAC,aAAa,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,CAAC;IAE/C,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;AACnC,CAAC;AAED,SAAS,GAAG,CAAC,IAAc;IACzB,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAChC,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC;AACvD,CAAC;AAED,SAAS,GAAG,CAAC,IAAc;IACzB,OAAO,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type Database from 'better-sqlite3';
|
|
2
|
+
import type { StructXConfig } from '../config';
|
|
3
|
+
export interface BenchmarkRunResult {
|
|
4
|
+
question: string;
|
|
5
|
+
structx: {
|
|
6
|
+
answer: string;
|
|
7
|
+
inputTokens: number;
|
|
8
|
+
outputTokens: number;
|
|
9
|
+
cost: number;
|
|
10
|
+
responseTimeMs: number;
|
|
11
|
+
functionsRetrieved: number;
|
|
12
|
+
graphQueryTimeMs: number;
|
|
13
|
+
} | null;
|
|
14
|
+
traditional: {
|
|
15
|
+
answer: string;
|
|
16
|
+
inputTokens: number;
|
|
17
|
+
outputTokens: number;
|
|
18
|
+
cost: number;
|
|
19
|
+
responseTimeMs: number;
|
|
20
|
+
filesAccessed: number;
|
|
21
|
+
} | null;
|
|
22
|
+
}
|
|
23
|
+
export declare function runBenchmark(db: Database.Database, config: StructXConfig, questions?: string[]): Promise<BenchmarkRunResult[]>;
|
|
24
|
+
//# sourceMappingURL=runner.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,QAAQ,MAAM,gBAAgB,CAAC;AAC3C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,WAAW,CAAC;AAU/C,MAAM,WAAW,kBAAkB;IACjC,QAAQ,EAAE,MAAM,CAAC;IACjB,OAAO,EAAE;QACP,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,cAAc,EAAE,MAAM,CAAC;QACvB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,gBAAgB,EAAE,MAAM,CAAC;KAC1B,GAAG,IAAI,CAAC;IACT,WAAW,EAAE;QACX,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;QACrB,IAAI,EAAE,MAAM,CAAC;QACb,cAAc,EAAE,MAAM,CAAC;QACvB,aAAa,EAAE,MAAM,CAAC;KACvB,GAAG,IAAI,CAAC;CACV;AAED,wBAAsB,YAAY,CAChC,EAAE,EAAE,QAAQ,CAAC,QAAQ,EACrB,MAAM,EAAE,aAAa,EACrB,SAAS,CAAC,EAAE,MAAM,EAAE,GACnB,OAAO,CAAC,kBAAkB,EAAE,CAAC,CA6G/B"}
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.runBenchmark = runBenchmark;
|
|
4
|
+
const questions_1 = require("./questions");
|
|
5
|
+
const baseline_1 = require("./baseline");
|
|
6
|
+
const classifier_1 = require("../query/classifier");
|
|
7
|
+
const retriever_1 = require("../query/retriever");
|
|
8
|
+
const context_builder_1 = require("../query/context-builder");
|
|
9
|
+
const answerer_1 = require("../query/answerer");
|
|
10
|
+
const queries_1 = require("../db/queries");
|
|
11
|
+
const logger_1 = require("../utils/logger");
|
|
12
|
+
async function runBenchmark(db, config, questions) {
|
|
13
|
+
const questionList = questions || questions_1.BENCHMARK_QUESTIONS;
|
|
14
|
+
const results = [];
|
|
15
|
+
for (let i = 0; i < questionList.length; i++) {
|
|
16
|
+
const question = questionList[i];
|
|
17
|
+
console.log(`\n[${i + 1}/${questionList.length}] "${question}"`);
|
|
18
|
+
const result = { question, structx: null, traditional: null };
|
|
19
|
+
// Run StructX agent
|
|
20
|
+
try {
|
|
21
|
+
console.log(' Running StructX agent...');
|
|
22
|
+
const classification = await (0, classifier_1.classifyQuestion)(question, config.classifierModel, config.anthropicApiKey);
|
|
23
|
+
const graphQueryStart = Date.now();
|
|
24
|
+
let retrieved;
|
|
25
|
+
switch (classification.strategy) {
|
|
26
|
+
case 'direct':
|
|
27
|
+
retrieved = (0, retriever_1.directLookup)(db, classification.functionName || '');
|
|
28
|
+
break;
|
|
29
|
+
case 'relationship':
|
|
30
|
+
retrieved = (0, retriever_1.relationshipQuery)(db, classification.functionName || '', classification.direction || 'callers');
|
|
31
|
+
break;
|
|
32
|
+
case 'semantic':
|
|
33
|
+
retrieved = (0, retriever_1.semanticSearch)(db, classification.keywords);
|
|
34
|
+
break;
|
|
35
|
+
case 'domain':
|
|
36
|
+
retrieved = (0, retriever_1.domainQuery)(db, classification.domain || 'other');
|
|
37
|
+
break;
|
|
38
|
+
case 'impact':
|
|
39
|
+
retrieved = (0, retriever_1.impactAnalysis)(db, classification.functionName || '');
|
|
40
|
+
break;
|
|
41
|
+
default:
|
|
42
|
+
retrieved = (0, retriever_1.semanticSearch)(db, classification.keywords);
|
|
43
|
+
}
|
|
44
|
+
const graphQueryTimeMs = Date.now() - graphQueryStart;
|
|
45
|
+
const context = (0, context_builder_1.buildContext)(retrieved, question);
|
|
46
|
+
const answerResult = await (0, answerer_1.generateAnswer)(question, context, config.answerModel, config.anthropicApiKey);
|
|
47
|
+
result.structx = {
|
|
48
|
+
answer: answerResult.answer,
|
|
49
|
+
inputTokens: answerResult.inputTokens,
|
|
50
|
+
outputTokens: answerResult.outputTokens,
|
|
51
|
+
cost: answerResult.cost,
|
|
52
|
+
responseTimeMs: answerResult.responseTimeMs,
|
|
53
|
+
functionsRetrieved: retrieved.functions.length,
|
|
54
|
+
graphQueryTimeMs,
|
|
55
|
+
};
|
|
56
|
+
(0, queries_1.insertQaRun)(db, {
|
|
57
|
+
mode: 'structx',
|
|
58
|
+
question,
|
|
59
|
+
input_tokens: answerResult.inputTokens,
|
|
60
|
+
output_tokens: answerResult.outputTokens,
|
|
61
|
+
total_tokens: answerResult.inputTokens + answerResult.outputTokens,
|
|
62
|
+
cost_usd: answerResult.cost,
|
|
63
|
+
response_time_ms: answerResult.responseTimeMs,
|
|
64
|
+
files_accessed: null,
|
|
65
|
+
functions_retrieved: retrieved.functions.length,
|
|
66
|
+
graph_query_time_ms: graphQueryTimeMs,
|
|
67
|
+
answer_text: answerResult.answer,
|
|
68
|
+
});
|
|
69
|
+
console.log(` StructX: ${answerResult.inputTokens} in / ${answerResult.outputTokens} out | $${answerResult.cost.toFixed(4)} | ${answerResult.responseTimeMs}ms`);
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
logger_1.logger.error(`StructX agent failed: ${err.message}`);
|
|
73
|
+
console.log(` StructX: FAILED - ${err.message}`);
|
|
74
|
+
}
|
|
75
|
+
// Run Traditional agent
|
|
76
|
+
try {
|
|
77
|
+
console.log(' Running Traditional agent...');
|
|
78
|
+
const baseline = await (0, baseline_1.runBaseline)(question, config.repoPath, config.answerModel, config.anthropicApiKey);
|
|
79
|
+
result.traditional = {
|
|
80
|
+
answer: baseline.answer,
|
|
81
|
+
inputTokens: baseline.inputTokens,
|
|
82
|
+
outputTokens: baseline.outputTokens,
|
|
83
|
+
cost: baseline.cost,
|
|
84
|
+
responseTimeMs: baseline.responseTimeMs,
|
|
85
|
+
filesAccessed: baseline.filesAccessed,
|
|
86
|
+
};
|
|
87
|
+
(0, queries_1.insertQaRun)(db, {
|
|
88
|
+
mode: 'traditional',
|
|
89
|
+
question,
|
|
90
|
+
input_tokens: baseline.inputTokens,
|
|
91
|
+
output_tokens: baseline.outputTokens,
|
|
92
|
+
total_tokens: baseline.inputTokens + baseline.outputTokens,
|
|
93
|
+
cost_usd: baseline.cost,
|
|
94
|
+
response_time_ms: baseline.responseTimeMs,
|
|
95
|
+
files_accessed: baseline.filesAccessed,
|
|
96
|
+
functions_retrieved: null,
|
|
97
|
+
graph_query_time_ms: null,
|
|
98
|
+
answer_text: baseline.answer,
|
|
99
|
+
});
|
|
100
|
+
console.log(` Traditional: ${baseline.inputTokens} in / ${baseline.outputTokens} out | $${baseline.cost.toFixed(4)} | ${baseline.responseTimeMs}ms`);
|
|
101
|
+
}
|
|
102
|
+
catch (err) {
|
|
103
|
+
logger_1.logger.error(`Traditional agent failed: ${err.message}`);
|
|
104
|
+
console.log(` Traditional: FAILED - ${err.message}`);
|
|
105
|
+
}
|
|
106
|
+
results.push(result);
|
|
107
|
+
}
|
|
108
|
+
return results;
|
|
109
|
+
}
|
|
110
|
+
//# sourceMappingURL=runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner.js","sourceRoot":"","sources":["../../src/benchmark/runner.ts"],"names":[],"mappings":";;AAgCA,oCAiHC;AA/ID,2CAAkD;AAClD,yCAAyC;AACzC,oDAAuD;AACvD,kDAAkH;AAClH,8DAAwD;AACxD,gDAAmD;AACnD,2CAA4C;AAC5C,4CAAyC;AAuBlC,KAAK,UAAU,YAAY,CAChC,EAAqB,EACrB,MAAqB,EACrB,SAAoB;IAEpB,MAAM,YAAY,GAAG,SAAS,IAAI,+BAAmB,CAAC;IACtD,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7C,MAAM,QAAQ,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,YAAY,CAAC,MAAM,MAAM,QAAQ,GAAG,CAAC,CAAC;QAEjE,MAAM,MAAM,GAAuB,EAAE,QAAQ,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;QAElF,oBAAoB;QACpB,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,4BAA4B,CAAC,CAAC;YAC1C,MAAM,cAAc,GAAG,MAAM,IAAA,6BAAgB,EAAC,QAAQ,EAAE,MAAM,CAAC,eAAe,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;YAExG,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACnC,IAAI,SAAS,CAAC;YACd,QAAQ,cAAc,CAAC,QAAQ,EAAE,CAAC;gBAChC,KAAK,QAAQ;oBACX,SAAS,GAAG,IAAA,wBAAY,EAAC,EAAE,EAAE,cAAc,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;oBAChE,MAAM;gBACR,KAAK,cAAc;oBACjB,SAAS,GAAG,IAAA,6BAAiB,EAAC,EAAE,EAAE,cAAc,CAAC,YAAY,IAAI,EAAE,EAAE,cAAc,CAAC,SAAS,IAAI,SAAS,CAAC,CAAC;oBAC5G,MAAM;gBACR,KAAK,UAAU;oBACb,SAAS,GAAG,IAAA,0BAAc,EAAC,EAAE,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;oBACxD,MAAM;gBACR,KAAK,QAAQ;oBACX,SAAS,GAAG,IAAA,uBAAW,EAAC,EAAE,EAAE,cAAc,CAAC,MAAM,IAAI,OAAO,CAAC,CAAC;oBAC9D,MAAM;gBACR,KAAK,QAAQ;oBACX,SAAS,GAAG,IAAA,0BAAc,EAAC,EAAE,EAAE,cAAc,CAAC,YAAY,IAAI,EAAE,CAAC,CAAC;oBAClE,MAAM;gBACR;oBACE,SAAS,GAAG,IAAA,0BAAc,EAAC,EAAE,EAAE,cAAc,CAAC,QAAQ,CAAC,CAAC;YAC5D,CAAC;YACD,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,eAAe,CAAC;YAEtD,MAAM,OAAO,GAAG,IAAA,8BAAY,EAAC,SAAS,EAAE,QAAQ,CAAC,CAAC;YAClD,MAAM,YAAY,GAAG,MAAM,IAAA,yBAAc,EAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;YAEzG,MAAM,CAAC,OAAO,GAAG;gBACf,MAAM,EAAE,YAAY,CAAC,MAAM;gBAC3B,WAAW,EAAE,YAAY,CAAC,WAAW;gBACrC,YAAY,EAAE,YAAY,CAAC,YAAY;gBACvC,IAAI,EAAE,YAAY,CAAC,IAAI;gBACvB,cAAc,EAAE,YAAY,CAAC,cAAc;gBAC3C,kBAAkB,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM;gBAC9C,gBAAgB;aACjB,CAAC;YAEF,IAAA,qBAAW,EAAC,EAAE,EAAE;gBACd,IAAI,EAAE,SAAS;gBACf,QAAQ;gBACR,YAAY,EAAE,YAAY,CAAC,WAAW;gBACtC,aAAa,EAAE,YAAY,CAAC,YAAY;gBACxC,YAAY,EAAE,YAAY,CAAC,WAAW,GAAG,YAAY,CAAC,YAAY;gBAClE,QAAQ,EAAE,YAAY,CAAC,IAAI;gBAC3B,gBAAgB,EAAE,YAAY,CAAC,cAAc;gBAC7C,cAAc,EAAE,IAAI;gBACpB,mBAAmB,EAAE,SAAS,CAAC,SAAS,CAAC,MAAM;gBAC/C,mBAAmB,EAAE,gBAAgB;gBACrC,WAAW,EAAE,YAAY,CAAC,MAAM;aACjC,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,cAAc,YAAY,CAAC,WAAW,SAAS,YAAY,CAAC,YAAY,WAAW,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,cAAc,IAAI,CAAC,CAAC;QACpK,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,eAAM,CAAC,KAAK,CAAC,yBAAyB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACrD,OAAO,CAAC,GAAG,CAAC,uBAAuB,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACpD,CAAC;QAED,wBAAwB;QACxB,IAAI,CAAC;YACH,OAAO,CAAC,GAAG,CAAC,gCAAgC,CAAC,CAAC;YAC9C,MAAM,QAAQ,GAAG,MAAM,IAAA,sBAAW,EAAC,QAAQ,EAAE,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,WAAW,EAAE,MAAM,CAAC,eAAe,CAAC,CAAC;YAE1G,MAAM,CAAC,WAAW,GAAG;gBACnB,MAAM,EAAE,QAAQ,CAAC,MAAM;gBACvB,WAAW,EAAE,QAAQ,CAAC,WAAW;gBACjC,YAAY,EAAE,QAAQ,CAAC,YAAY;gBACnC,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,cAAc,EAAE,QAAQ,CAAC,cAAc;gBACvC,aAAa,EAAE,QAAQ,CAAC,aAAa;aACtC,CAAC;YAEF,IAAA,qBAAW,EAAC,EAAE,EAAE;gBACd,IAAI,EAAE,aAAa;gBACnB,QAAQ;gBACR,YAAY,EAAE,QAAQ,CAAC,WAAW;gBAClC,aAAa,EAAE,QAAQ,CAAC,YAAY;gBACpC,YAAY,EAAE,QAAQ,CAAC,WAAW,GAAG,QAAQ,CAAC,YAAY;gBAC1D,QAAQ,EAAE,QAAQ,CAAC,IAAI;gBACvB,gBAAgB,EAAE,QAAQ,CAAC,cAAc;gBACzC,cAAc,EAAE,QAAQ,CAAC,aAAa;gBACtC,mBAAmB,EAAE,IAAI;gBACzB,mBAAmB,EAAE,IAAI;gBACzB,WAAW,EAAE,QAAQ,CAAC,MAAM;aAC7B,CAAC,CAAC;YAEH,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,CAAC,WAAW,SAAS,QAAQ,CAAC,YAAY,WAAW,QAAQ,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,QAAQ,CAAC,cAAc,IAAI,CAAC,CAAC;QACxJ,CAAC;QAAC,OAAO,GAAQ,EAAE,CAAC;YAClB,eAAM,CAAC,KAAK,CAAC,6BAA6B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;YACzD,OAAO,CAAC,GAAG,CAAC,2BAA2B,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACvB,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli.d.ts","sourceRoot":"","sources":["../src/cli.ts"],"names":[],"mappings":";AAEA,OAAO,eAAe,CAAC"}
|