tryassay 0.20.3 → 0.21.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/server.d.ts +4 -0
- package/dist/api/server.js +247 -5
- package/dist/api/server.js.map +1 -1
- package/dist/cli.js +3 -1
- package/dist/cli.js.map +1 -1
- package/dist/commands/create.d.ts +2 -0
- package/dist/commands/create.js +74 -0
- package/dist/commands/create.js.map +1 -1
- package/dist/runtime/__tests__/check-loop.test.js +2 -2
- package/dist/runtime/__tests__/cross-verification-checks.test.d.ts +8 -0
- package/dist/runtime/__tests__/cross-verification-checks.test.js +365 -0
- package/dist/runtime/__tests__/cross-verification-checks.test.js.map +1 -0
- package/dist/runtime/agents/planner-agent.d.ts +18 -1
- package/dist/runtime/agents/planner-agent.js +162 -0
- package/dist/runtime/agents/planner-agent.js.map +1 -1
- package/dist/runtime/app-create-orchestrator.d.ts +46 -1
- package/dist/runtime/app-create-orchestrator.js +691 -7
- package/dist/runtime/app-create-orchestrator.js.map +1 -1
- package/dist/runtime/check-catalog.js +53 -0
- package/dist/runtime/check-catalog.js.map +1 -1
- package/dist/runtime/fs-helpers.d.ts +2 -0
- package/dist/runtime/fs-helpers.js +14 -0
- package/dist/runtime/fs-helpers.js.map +1 -1
- package/dist/runtime/functional-tester.d.ts +47 -0
- package/dist/runtime/functional-tester.js +775 -0
- package/dist/runtime/functional-tester.js.map +1 -0
- package/dist/runtime/plan-refiner.d.ts +14 -0
- package/dist/runtime/plan-refiner.js +160 -0
- package/dist/runtime/plan-refiner.js.map +1 -0
- package/dist/runtime/types.d.ts +126 -1
- package/package.json +1 -1
|
@@ -0,0 +1,775 @@
|
|
|
1
|
+
// ============================================================
|
|
2
|
+
// Functional Tester — HTTP-level testing of generated apps
|
|
3
|
+
// Starts the dev server, generates tests from the architecture plan,
|
|
4
|
+
// runs them via fetch(), and feeds failures to CodeAgent for repair.
|
|
5
|
+
// ============================================================
|
|
6
|
+
import { spawn } from 'node:child_process';
|
|
7
|
+
import { readFile, writeFile, mkdir } from 'node:fs/promises';
|
|
8
|
+
import { join, dirname } from 'node:path';
|
|
9
|
+
import { randomUUID } from 'node:crypto';
|
|
10
|
+
import { CodeAgent } from './agents/code-agent.js';
|
|
11
|
+
import { MessageBus } from './message-bus.js';
|
|
12
|
+
// ── Functional Tester ────────────────────────────────────────
|
|
13
|
+
export class FunctionalTester {
|
|
14
|
+
projectPath;
|
|
15
|
+
plan;
|
|
16
|
+
maxRepairAttempts;
|
|
17
|
+
testTimeoutMs;
|
|
18
|
+
serverStartTimeoutMs;
|
|
19
|
+
codeModel;
|
|
20
|
+
framework;
|
|
21
|
+
constructor(projectPath, plan, options) {
|
|
22
|
+
this.projectPath = projectPath;
|
|
23
|
+
this.plan = plan;
|
|
24
|
+
this.maxRepairAttempts = options?.maxRepairAttempts ?? 3;
|
|
25
|
+
this.testTimeoutMs = options?.testTimeoutMs ?? 10_000;
|
|
26
|
+
this.serverStartTimeoutMs = options?.serverStartTimeoutMs ?? 30_000;
|
|
27
|
+
this.codeModel = options?.codeModel;
|
|
28
|
+
this.framework = (options?.framework ?? 'next.js').toLowerCase();
|
|
29
|
+
}
|
|
30
|
+
/** Run functional tests with repair loop. */
|
|
31
|
+
async test() {
|
|
32
|
+
const start = Date.now();
|
|
33
|
+
// Skip for Electron (no HTTP server)
|
|
34
|
+
if (this.framework === 'electron') {
|
|
35
|
+
return {
|
|
36
|
+
status: 'skipped',
|
|
37
|
+
tests: [],
|
|
38
|
+
passedCount: 0,
|
|
39
|
+
failedCount: 0,
|
|
40
|
+
repairAttempts: [],
|
|
41
|
+
serverPort: null,
|
|
42
|
+
totalDurationMs: Date.now() - start,
|
|
43
|
+
};
|
|
44
|
+
}
|
|
45
|
+
// Generate test cases from the plan
|
|
46
|
+
const tests = this.generateTests();
|
|
47
|
+
if (tests.length === 0) {
|
|
48
|
+
return {
|
|
49
|
+
status: 'skipped',
|
|
50
|
+
tests: [],
|
|
51
|
+
passedCount: 0,
|
|
52
|
+
failedCount: 0,
|
|
53
|
+
repairAttempts: [],
|
|
54
|
+
serverPort: null,
|
|
55
|
+
totalDurationMs: Date.now() - start,
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
// Start dev server
|
|
59
|
+
let serverProcess = null;
|
|
60
|
+
let port = null;
|
|
61
|
+
try {
|
|
62
|
+
const serverInfo = await this.startDevServer();
|
|
63
|
+
serverProcess = serverInfo.process;
|
|
64
|
+
port = serverInfo.port;
|
|
65
|
+
}
|
|
66
|
+
catch (err) {
|
|
67
|
+
// Server failed to start — return fail with no tests run
|
|
68
|
+
return {
|
|
69
|
+
status: 'fail',
|
|
70
|
+
tests: [],
|
|
71
|
+
passedCount: 0,
|
|
72
|
+
failedCount: 0,
|
|
73
|
+
repairAttempts: [],
|
|
74
|
+
serverPort: null,
|
|
75
|
+
totalDurationMs: Date.now() - start,
|
|
76
|
+
};
|
|
77
|
+
}
|
|
78
|
+
try {
|
|
79
|
+
// Run tests
|
|
80
|
+
let executions = await this.runTests(tests, port);
|
|
81
|
+
const repairAttempts = [];
|
|
82
|
+
let failures = executions.filter(e => e.status !== 'pass');
|
|
83
|
+
let attempt = 0;
|
|
84
|
+
// Repair loop
|
|
85
|
+
while (failures.length > 0 && attempt < this.maxRepairAttempts) {
|
|
86
|
+
attempt++;
|
|
87
|
+
const repairStart = Date.now();
|
|
88
|
+
const repairResult = await this.runRepair(failures, port);
|
|
89
|
+
repairAttempts.push({
|
|
90
|
+
attempt,
|
|
91
|
+
failures,
|
|
92
|
+
filesModified: repairResult.filesModified,
|
|
93
|
+
repairSucceeded: repairResult.filesModified.length > 0,
|
|
94
|
+
durationMs: Date.now() - repairStart,
|
|
95
|
+
});
|
|
96
|
+
if (repairResult.filesModified.length === 0) {
|
|
97
|
+
break; // Repair couldn't modify anything, stop trying
|
|
98
|
+
}
|
|
99
|
+
// Re-run only failed tests
|
|
100
|
+
const failedTests = failures.map(f => f.test);
|
|
101
|
+
executions = [
|
|
102
|
+
...executions.filter(e => e.status === 'pass'),
|
|
103
|
+
...await this.runTests(failedTests, port),
|
|
104
|
+
];
|
|
105
|
+
failures = executions.filter(e => e.status !== 'pass');
|
|
106
|
+
}
|
|
107
|
+
const passedCount = executions.filter(e => e.status === 'pass').length;
|
|
108
|
+
const failedCount = executions.filter(e => e.status !== 'pass').length;
|
|
109
|
+
const status = failedCount === 0 ? (repairAttempts.length > 0 ? 'repaired' : 'pass') : 'fail';
|
|
110
|
+
return {
|
|
111
|
+
status,
|
|
112
|
+
tests: executions,
|
|
113
|
+
passedCount,
|
|
114
|
+
failedCount,
|
|
115
|
+
repairAttempts,
|
|
116
|
+
serverPort: port,
|
|
117
|
+
totalDurationMs: Date.now() - start,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
finally {
|
|
121
|
+
// Always kill the server
|
|
122
|
+
if (serverProcess) {
|
|
123
|
+
serverProcess.kill('SIGTERM');
|
|
124
|
+
// Give it a moment, then force kill
|
|
125
|
+
setTimeout(() => {
|
|
126
|
+
if (serverProcess && !serverProcess.killed) {
|
|
127
|
+
serverProcess.kill('SIGKILL');
|
|
128
|
+
}
|
|
129
|
+
}, 3000);
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
// ── Test Generation ──────────────────────────────────────────
|
|
134
|
+
/** Generate test cases from the architecture plan. */
|
|
135
|
+
generateTests() {
|
|
136
|
+
const tests = [];
|
|
137
|
+
// API route tests
|
|
138
|
+
for (const route of this.plan.apiRoutes) {
|
|
139
|
+
const body = route.method !== 'GET' && route.method !== 'DELETE'
|
|
140
|
+
? this.generateSamplePayload(route.path, route.featureId)
|
|
141
|
+
: undefined;
|
|
142
|
+
tests.push({
|
|
143
|
+
id: `api-${route.method.toLowerCase()}-${route.path.replace(/\//g, '-')}`,
|
|
144
|
+
type: 'api_route',
|
|
145
|
+
name: `${route.method} ${route.path}`,
|
|
146
|
+
method: route.method,
|
|
147
|
+
path: route.path,
|
|
148
|
+
body,
|
|
149
|
+
expectedStatusRange: [200, 499], // 2xx-4xx are valid responses; 5xx = server error
|
|
150
|
+
errorPatterns: [
|
|
151
|
+
'Internal Server Error',
|
|
152
|
+
'TypeError',
|
|
153
|
+
'ReferenceError',
|
|
154
|
+
'Cannot read properties of',
|
|
155
|
+
'is not a function',
|
|
156
|
+
'ECONNREFUSED',
|
|
157
|
+
'MODULE_NOT_FOUND',
|
|
158
|
+
],
|
|
159
|
+
featureId: route.featureId,
|
|
160
|
+
});
|
|
161
|
+
}
|
|
162
|
+
// Page load tests
|
|
163
|
+
for (const page of this.plan.pages) {
|
|
164
|
+
tests.push({
|
|
165
|
+
id: `page-${page.path.replace(/\//g, '-') || 'root'}`,
|
|
166
|
+
type: 'page_load',
|
|
167
|
+
name: `GET ${page.path || '/'}`,
|
|
168
|
+
method: 'GET',
|
|
169
|
+
path: page.path || '/',
|
|
170
|
+
expectedStatusRange: [200, 399], // Pages should return 2xx or redirect
|
|
171
|
+
errorPatterns: [
|
|
172
|
+
'Internal Server Error',
|
|
173
|
+
'Application error',
|
|
174
|
+
'Error: ',
|
|
175
|
+
'TypeError',
|
|
176
|
+
'Unhandled Runtime Error',
|
|
177
|
+
'Module not found',
|
|
178
|
+
'500',
|
|
179
|
+
],
|
|
180
|
+
featureId: page.featureId,
|
|
181
|
+
});
|
|
182
|
+
}
|
|
183
|
+
// Auth flow tests (if auth is configured)
|
|
184
|
+
if (this.plan.authPlan) {
|
|
185
|
+
// Test signup route exists and doesn't crash
|
|
186
|
+
const signupRoutes = this.plan.apiRoutes.filter(r => r.path.includes('signup') || r.path.includes('register'));
|
|
187
|
+
for (const route of signupRoutes) {
|
|
188
|
+
tests.push({
|
|
189
|
+
id: `auth-signup-${route.path.replace(/\//g, '-')}`,
|
|
190
|
+
type: 'auth_flow',
|
|
191
|
+
name: `Auth: POST ${route.path}`,
|
|
192
|
+
method: 'POST',
|
|
193
|
+
path: route.path,
|
|
194
|
+
body: {
|
|
195
|
+
email: 'test@example.com',
|
|
196
|
+
password: 'TestPassword123!',
|
|
197
|
+
},
|
|
198
|
+
expectedStatusRange: [200, 499], // Auth may return 4xx for duplicate etc
|
|
199
|
+
errorPatterns: [
|
|
200
|
+
'Internal Server Error',
|
|
201
|
+
'TypeError',
|
|
202
|
+
'Cannot read properties of',
|
|
203
|
+
'ECONNREFUSED',
|
|
204
|
+
],
|
|
205
|
+
featureId: route.featureId,
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
// Test login route
|
|
209
|
+
const loginRoutes = this.plan.apiRoutes.filter(r => r.path.includes('login') || r.path.includes('signin'));
|
|
210
|
+
for (const route of loginRoutes) {
|
|
211
|
+
tests.push({
|
|
212
|
+
id: `auth-login-${route.path.replace(/\//g, '-')}`,
|
|
213
|
+
type: 'auth_flow',
|
|
214
|
+
name: `Auth: POST ${route.path}`,
|
|
215
|
+
method: 'POST',
|
|
216
|
+
path: route.path,
|
|
217
|
+
body: {
|
|
218
|
+
email: 'test@example.com',
|
|
219
|
+
password: 'TestPassword123!',
|
|
220
|
+
},
|
|
221
|
+
expectedStatusRange: [200, 499],
|
|
222
|
+
errorPatterns: [
|
|
223
|
+
'Internal Server Error',
|
|
224
|
+
'TypeError',
|
|
225
|
+
'Cannot read properties of',
|
|
226
|
+
'ECONNREFUSED',
|
|
227
|
+
],
|
|
228
|
+
featureId: route.featureId,
|
|
229
|
+
});
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
// Data persistence tests: POST → GET to verify data was saved
|
|
233
|
+
for (const route of this.plan.apiRoutes) {
|
|
234
|
+
if (route.method !== 'POST')
|
|
235
|
+
continue;
|
|
236
|
+
const getRoute = this.findCorrespondingGetRoute(route.path, this.plan.apiRoutes);
|
|
237
|
+
if (!getRoute)
|
|
238
|
+
continue;
|
|
239
|
+
const body = this.generateSamplePayload(route.path, route.featureId);
|
|
240
|
+
tests.push({
|
|
241
|
+
id: `persist-${route.path.replace(/\//g, '-')}`,
|
|
242
|
+
type: 'data_persistence',
|
|
243
|
+
name: `Persist: POST ${route.path} → GET ${getRoute}`,
|
|
244
|
+
method: 'POST',
|
|
245
|
+
path: route.path,
|
|
246
|
+
body,
|
|
247
|
+
expectedStatusRange: [200, 499],
|
|
248
|
+
errorPatterns: [
|
|
249
|
+
'Internal Server Error',
|
|
250
|
+
'TypeError',
|
|
251
|
+
'ReferenceError',
|
|
252
|
+
'Cannot read properties of',
|
|
253
|
+
],
|
|
254
|
+
featureId: route.featureId,
|
|
255
|
+
verifyPath: getRoute,
|
|
256
|
+
idField: this.guessIdField(route.path),
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
return tests;
|
|
260
|
+
}
|
|
261
|
+
/** Generate a minimal sample payload for a POST/PUT route based on schema. */
|
|
262
|
+
generateSamplePayload(routePath, featureId) {
|
|
263
|
+
// Try to find the schema entity for this feature
|
|
264
|
+
if (featureId) {
|
|
265
|
+
const feature = this.plan.features.find(f => f.id === featureId);
|
|
266
|
+
if (feature && feature.schemaEntities.length > 0) {
|
|
267
|
+
const entityName = feature.schemaEntities[0];
|
|
268
|
+
const entity = this.plan.schema.find(e => e.name === entityName);
|
|
269
|
+
if (entity) {
|
|
270
|
+
const payload = {};
|
|
271
|
+
for (const field of entity.fields) {
|
|
272
|
+
if (field.primaryKey)
|
|
273
|
+
continue; // Skip auto-generated IDs
|
|
274
|
+
if (field.name === 'created_at' || field.name === 'updated_at')
|
|
275
|
+
continue;
|
|
276
|
+
payload[field.name] = this.sampleValueForType(field.type, field.name);
|
|
277
|
+
}
|
|
278
|
+
return payload;
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
// Fallback: extract resource name from path
|
|
283
|
+
const parts = routePath.split('/').filter(Boolean);
|
|
284
|
+
const resource = parts[parts.length - 1] ?? 'item';
|
|
285
|
+
return {
|
|
286
|
+
name: `Test ${resource}`,
|
|
287
|
+
description: `Test ${resource} description`,
|
|
288
|
+
};
|
|
289
|
+
}
|
|
290
|
+
/** Generate a sample value for a schema field type. */
|
|
291
|
+
sampleValueForType(type, name) {
|
|
292
|
+
const t = type.toLowerCase();
|
|
293
|
+
if (t.includes('int') || t.includes('number') || t.includes('decimal') || t.includes('float'))
|
|
294
|
+
return 42;
|
|
295
|
+
if (t.includes('bool'))
|
|
296
|
+
return true;
|
|
297
|
+
if (t.includes('date') || t.includes('timestamp'))
|
|
298
|
+
return new Date().toISOString();
|
|
299
|
+
if (t.includes('json') || t.includes('object'))
|
|
300
|
+
return {};
|
|
301
|
+
if (t.includes('array'))
|
|
302
|
+
return [];
|
|
303
|
+
// String types
|
|
304
|
+
if (name.includes('email'))
|
|
305
|
+
return 'test@example.com';
|
|
306
|
+
if (name.includes('url'))
|
|
307
|
+
return 'https://example.com';
|
|
308
|
+
if (name.includes('phone'))
|
|
309
|
+
return '+12025551234';
|
|
310
|
+
return `test-${name}`;
|
|
311
|
+
}
|
|
312
|
+
// ── Server Management ────────────────────────────────────────
|
|
313
|
+
/** Start the dev server and detect the port. */
|
|
314
|
+
startDevServer() {
|
|
315
|
+
return new Promise((resolve, reject) => {
|
|
316
|
+
const command = this.framework.includes('next') ? 'npx' : 'npm';
|
|
317
|
+
const args = this.framework.includes('next')
|
|
318
|
+
? ['next', 'dev', '--port', '0'] // port 0 = auto-assign
|
|
319
|
+
: ['run', 'dev'];
|
|
320
|
+
const proc = spawn(command, args, {
|
|
321
|
+
cwd: this.projectPath,
|
|
322
|
+
env: { ...process.env, NODE_ENV: 'development', PORT: '0' },
|
|
323
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
324
|
+
});
|
|
325
|
+
let stdout = '';
|
|
326
|
+
let stderr = '';
|
|
327
|
+
let resolved = false;
|
|
328
|
+
const timeout = setTimeout(() => {
|
|
329
|
+
if (!resolved) {
|
|
330
|
+
resolved = true;
|
|
331
|
+
proc.kill('SIGTERM');
|
|
332
|
+
reject(new Error(`Server failed to start within ${this.serverStartTimeoutMs}ms. stdout: ${stdout.slice(-500)}, stderr: ${stderr.slice(-500)}`));
|
|
333
|
+
}
|
|
334
|
+
}, this.serverStartTimeoutMs);
|
|
335
|
+
const tryExtractPort = (data) => {
|
|
336
|
+
if (resolved)
|
|
337
|
+
return;
|
|
338
|
+
// Match common patterns: "localhost:3000", "port 3000", ":3000"
|
|
339
|
+
const portMatch = data.match(/(?:localhost|127\.0\.0\.1|0\.0\.0\.0):(\d+)/i)
|
|
340
|
+
?? data.match(/port\s+(\d+)/i)
|
|
341
|
+
?? data.match(/:(\d{4,5})\b/);
|
|
342
|
+
if (portMatch) {
|
|
343
|
+
const port = parseInt(portMatch[1], 10);
|
|
344
|
+
if (port > 0 && port < 65536) {
|
|
345
|
+
resolved = true;
|
|
346
|
+
clearTimeout(timeout);
|
|
347
|
+
resolve({ process: proc, port });
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
};
|
|
351
|
+
proc.stdout?.on('data', (chunk) => {
|
|
352
|
+
const data = chunk.toString();
|
|
353
|
+
stdout += data;
|
|
354
|
+
tryExtractPort(data);
|
|
355
|
+
});
|
|
356
|
+
proc.stderr?.on('data', (chunk) => {
|
|
357
|
+
const data = chunk.toString();
|
|
358
|
+
stderr += data;
|
|
359
|
+
tryExtractPort(data);
|
|
360
|
+
});
|
|
361
|
+
proc.on('error', (err) => {
|
|
362
|
+
if (!resolved) {
|
|
363
|
+
resolved = true;
|
|
364
|
+
clearTimeout(timeout);
|
|
365
|
+
reject(err);
|
|
366
|
+
}
|
|
367
|
+
});
|
|
368
|
+
proc.on('exit', (code) => {
|
|
369
|
+
if (!resolved) {
|
|
370
|
+
resolved = true;
|
|
371
|
+
clearTimeout(timeout);
|
|
372
|
+
reject(new Error(`Server exited with code ${code} before starting. stderr: ${stderr.slice(-500)}`));
|
|
373
|
+
}
|
|
374
|
+
});
|
|
375
|
+
});
|
|
376
|
+
}
|
|
377
|
+
// ── Test Execution ───────────────────────────────────────────
|
|
378
|
+
/** Run a batch of functional tests against the running server. */
|
|
379
|
+
async runTests(tests, port) {
|
|
380
|
+
const results = [];
|
|
381
|
+
for (const test of tests) {
|
|
382
|
+
const execution = await this.runSingleTest(test, port);
|
|
383
|
+
results.push(execution);
|
|
384
|
+
}
|
|
385
|
+
return results;
|
|
386
|
+
}
|
|
387
|
+
/** Run a single functional test. */
|
|
388
|
+
async runSingleTest(test, port) {
|
|
389
|
+
if (test.type === 'data_persistence') {
|
|
390
|
+
return this.runPersistenceVerification(test, port);
|
|
391
|
+
}
|
|
392
|
+
const url = `http://localhost:${port}${test.path}`;
|
|
393
|
+
const start = Date.now();
|
|
394
|
+
try {
|
|
395
|
+
const controller = new AbortController();
|
|
396
|
+
const timeoutId = setTimeout(() => controller.abort(), this.testTimeoutMs);
|
|
397
|
+
const fetchOptions = {
|
|
398
|
+
method: test.method ?? 'GET',
|
|
399
|
+
signal: controller.signal,
|
|
400
|
+
headers: {
|
|
401
|
+
'Content-Type': 'application/json',
|
|
402
|
+
'Accept': 'application/json, text/html',
|
|
403
|
+
},
|
|
404
|
+
};
|
|
405
|
+
if (test.body && test.method !== 'GET') {
|
|
406
|
+
fetchOptions.body = JSON.stringify(test.body);
|
|
407
|
+
}
|
|
408
|
+
const response = await fetch(url, fetchOptions);
|
|
409
|
+
clearTimeout(timeoutId);
|
|
410
|
+
const statusCode = response.status;
|
|
411
|
+
let responseBody;
|
|
412
|
+
try {
|
|
413
|
+
responseBody = await response.text();
|
|
414
|
+
}
|
|
415
|
+
catch {
|
|
416
|
+
responseBody = '';
|
|
417
|
+
}
|
|
418
|
+
// Check status code range
|
|
419
|
+
const [minStatus, maxStatus] = test.expectedStatusRange;
|
|
420
|
+
if (statusCode < minStatus || statusCode > maxStatus) {
|
|
421
|
+
return {
|
|
422
|
+
test,
|
|
423
|
+
status: 'fail',
|
|
424
|
+
statusCode,
|
|
425
|
+
responseBody: responseBody.slice(0, 2000),
|
|
426
|
+
errorMatch: `Status ${statusCode} outside expected range [${minStatus}, ${maxStatus}]`,
|
|
427
|
+
durationMs: Date.now() - start,
|
|
428
|
+
};
|
|
429
|
+
}
|
|
430
|
+
// Check for 5xx errors specifically (server bugs)
|
|
431
|
+
if (statusCode >= 500) {
|
|
432
|
+
return {
|
|
433
|
+
test,
|
|
434
|
+
status: 'fail',
|
|
435
|
+
statusCode,
|
|
436
|
+
responseBody: responseBody.slice(0, 2000),
|
|
437
|
+
errorMatch: `Server error: ${statusCode}`,
|
|
438
|
+
durationMs: Date.now() - start,
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
// Check response body for error patterns
|
|
442
|
+
for (const pattern of test.errorPatterns) {
|
|
443
|
+
if (responseBody.includes(pattern)) {
|
|
444
|
+
return {
|
|
445
|
+
test,
|
|
446
|
+
status: 'fail',
|
|
447
|
+
statusCode,
|
|
448
|
+
responseBody: responseBody.slice(0, 2000),
|
|
449
|
+
errorMatch: `Response contains error pattern: "${pattern}"`,
|
|
450
|
+
durationMs: Date.now() - start,
|
|
451
|
+
};
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
return {
|
|
455
|
+
test,
|
|
456
|
+
status: 'pass',
|
|
457
|
+
statusCode,
|
|
458
|
+
responseBody: responseBody.slice(0, 500),
|
|
459
|
+
durationMs: Date.now() - start,
|
|
460
|
+
};
|
|
461
|
+
}
|
|
462
|
+
catch (err) {
|
|
463
|
+
if (err instanceof Error && err.name === 'AbortError') {
|
|
464
|
+
return {
|
|
465
|
+
test,
|
|
466
|
+
status: 'timeout',
|
|
467
|
+
statusCode: null,
|
|
468
|
+
responseBody: '',
|
|
469
|
+
errorMatch: `Request timed out after ${this.testTimeoutMs}ms`,
|
|
470
|
+
durationMs: Date.now() - start,
|
|
471
|
+
};
|
|
472
|
+
}
|
|
473
|
+
return {
|
|
474
|
+
test,
|
|
475
|
+
status: 'error',
|
|
476
|
+
statusCode: null,
|
|
477
|
+
responseBody: '',
|
|
478
|
+
errorMatch: err instanceof Error ? err.message : String(err),
|
|
479
|
+
durationMs: Date.now() - start,
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
}
|
|
483
|
+
// ── Repair Loop ──────────────────────────────────────────────
|
|
484
|
+
/** Feed test failures to CodeAgent for repair. */
|
|
485
|
+
async runRepair(failures, port) {
|
|
486
|
+
const filesModified = [];
|
|
487
|
+
// Group failures by feature to batch repairs
|
|
488
|
+
const failuresByFeature = new Map();
|
|
489
|
+
for (const failure of failures) {
|
|
490
|
+
const featureId = failure.test.featureId ?? 'unknown';
|
|
491
|
+
const existing = failuresByFeature.get(featureId) ?? [];
|
|
492
|
+
existing.push(failure);
|
|
493
|
+
failuresByFeature.set(featureId, existing);
|
|
494
|
+
}
|
|
495
|
+
for (const [featureId, featureFailures] of failuresByFeature) {
|
|
496
|
+
// Build the repair goal with failure details
|
|
497
|
+
const failureDetails = featureFailures.map(f => {
|
|
498
|
+
return `- ${f.test.name}: ${f.status} — ${f.errorMatch ?? 'Unknown error'}
|
|
499
|
+
Status code: ${f.statusCode ?? 'N/A'}
|
|
500
|
+
Response: ${f.responseBody.slice(0, 500)}`;
|
|
501
|
+
}).join('\n');
|
|
502
|
+
// Try to read the source files for the failing routes/pages
|
|
503
|
+
const sourceContext = await this.readFailingSourceFiles(featureFailures);
|
|
504
|
+
const repairGoal = `Fix functional test failures in the generated application.
|
|
505
|
+
|
|
506
|
+
The following HTTP tests failed against the running dev server:
|
|
507
|
+
|
|
508
|
+
${failureDetails}
|
|
509
|
+
|
|
510
|
+
${sourceContext ? `Current source code for the failing endpoints:\n\n${sourceContext}` : ''}
|
|
511
|
+
|
|
512
|
+
Fix the code so these endpoints return valid responses without server errors.
|
|
513
|
+
Do NOT add placeholder stubs — implement the actual logic.
|
|
514
|
+
The dev server is running at http://localhost:${port}.`;
|
|
515
|
+
const messageBus = new MessageBus();
|
|
516
|
+
const codeAgent = new CodeAgent(messageBus, { model: this.codeModel });
|
|
517
|
+
try {
|
|
518
|
+
const result = await codeAgent.executeTask({
|
|
519
|
+
taskId: `repair-${featureId}-${randomUUID().slice(0, 8)}`,
|
|
520
|
+
goal: repairGoal,
|
|
521
|
+
constraints: [
|
|
522
|
+
`Framework: ${this.framework}`,
|
|
523
|
+
'Fix the actual bug — do not add stubs or TODO comments',
|
|
524
|
+
],
|
|
525
|
+
dependencies: [],
|
|
526
|
+
contextRefs: [],
|
|
527
|
+
});
|
|
528
|
+
if (result.status === 'completed' && result.artifacts.length > 0) {
|
|
529
|
+
for (const artifact of result.artifacts) {
|
|
530
|
+
if (!artifact.path)
|
|
531
|
+
continue;
|
|
532
|
+
const absPath = join(this.projectPath, artifact.path);
|
|
533
|
+
try {
|
|
534
|
+
await mkdir(dirname(absPath), { recursive: true });
|
|
535
|
+
await writeFile(absPath, artifact.content, 'utf-8');
|
|
536
|
+
filesModified.push(artifact.path);
|
|
537
|
+
}
|
|
538
|
+
catch {
|
|
539
|
+
// Best-effort write
|
|
540
|
+
}
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
}
|
|
544
|
+
catch {
|
|
545
|
+
// Repair agent failed — continue with remaining features
|
|
546
|
+
}
|
|
547
|
+
}
|
|
548
|
+
return { filesModified };
|
|
549
|
+
}
|
|
550
|
+
/** Read source files for failing test endpoints. */
|
|
551
|
+
async readFailingSourceFiles(failures) {
|
|
552
|
+
const sections = [];
|
|
553
|
+
for (const failure of failures) {
|
|
554
|
+
const path = failure.test.path;
|
|
555
|
+
const filePath = this.routeToFilePath(path);
|
|
556
|
+
try {
|
|
557
|
+
const content = await readFile(join(this.projectPath, filePath), 'utf-8');
|
|
558
|
+
sections.push(`--- ${filePath} ---\n${content}\n`);
|
|
559
|
+
}
|
|
560
|
+
catch {
|
|
561
|
+
// File doesn't exist
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
return sections.join('\n');
|
|
565
|
+
}
|
|
566
|
+
/** Map a route path to its expected file path (mirrors orchestrator logic). */
|
|
567
|
+
routeToFilePath(routePath) {
|
|
568
|
+
if (this.framework.includes('next')) {
|
|
569
|
+
return `app${routePath}/route.ts`;
|
|
570
|
+
}
|
|
571
|
+
const parts = routePath.replace(/^\/api/, '').replace(/^\//, '');
|
|
572
|
+
return `src/routes/${parts || 'index'}.ts`;
|
|
573
|
+
}
|
|
574
|
+
// ── Data Persistence Helpers ──────────────────────────────────
|
|
575
|
+
/** Run a persistence verification: POST data, then GET to confirm it was saved. */
|
|
576
|
+
async runPersistenceVerification(test, port) {
|
|
577
|
+
const baseUrl = `http://localhost:${port}`;
|
|
578
|
+
const start = Date.now();
|
|
579
|
+
try {
|
|
580
|
+
// Step 1: POST the data
|
|
581
|
+
const controller = new AbortController();
|
|
582
|
+
const timeoutId = setTimeout(() => controller.abort(), this.testTimeoutMs);
|
|
583
|
+
const postResponse = await fetch(`${baseUrl}${test.path}`, {
|
|
584
|
+
method: 'POST',
|
|
585
|
+
signal: controller.signal,
|
|
586
|
+
headers: {
|
|
587
|
+
'Content-Type': 'application/json',
|
|
588
|
+
'Accept': 'application/json',
|
|
589
|
+
},
|
|
590
|
+
body: JSON.stringify(test.body),
|
|
591
|
+
});
|
|
592
|
+
clearTimeout(timeoutId);
|
|
593
|
+
const postStatus = postResponse.status;
|
|
594
|
+
let postBody;
|
|
595
|
+
try {
|
|
596
|
+
postBody = await postResponse.text();
|
|
597
|
+
}
|
|
598
|
+
catch {
|
|
599
|
+
postBody = '';
|
|
600
|
+
}
|
|
601
|
+
// If POST itself fails with 5xx, report that
|
|
602
|
+
if (postStatus >= 500) {
|
|
603
|
+
return {
|
|
604
|
+
test,
|
|
605
|
+
status: 'fail',
|
|
606
|
+
statusCode: postStatus,
|
|
607
|
+
responseBody: postBody.slice(0, 2000),
|
|
608
|
+
errorMatch: `POST returned server error: ${postStatus}`,
|
|
609
|
+
durationMs: Date.now() - start,
|
|
610
|
+
};
|
|
611
|
+
}
|
|
612
|
+
// If POST returned a client error (4xx), it's not a persistence failure
|
|
613
|
+
if (postStatus >= 400) {
|
|
614
|
+
return {
|
|
615
|
+
test,
|
|
616
|
+
status: 'fail',
|
|
617
|
+
statusCode: postStatus,
|
|
618
|
+
responseBody: postBody.slice(0, 2000),
|
|
619
|
+
errorMatch: `POST returned client error: ${postStatus}`,
|
|
620
|
+
durationMs: Date.now() - start,
|
|
621
|
+
};
|
|
622
|
+
}
|
|
623
|
+
// Step 2: Extract ID from POST response
|
|
624
|
+
let postJson;
|
|
625
|
+
try {
|
|
626
|
+
postJson = JSON.parse(postBody);
|
|
627
|
+
}
|
|
628
|
+
catch {
|
|
629
|
+
return {
|
|
630
|
+
test,
|
|
631
|
+
status: 'fail',
|
|
632
|
+
statusCode: postStatus,
|
|
633
|
+
responseBody: postBody.slice(0, 2000),
|
|
634
|
+
errorMatch: 'POST response is not valid JSON — cannot extract ID',
|
|
635
|
+
durationMs: Date.now() - start,
|
|
636
|
+
};
|
|
637
|
+
}
|
|
638
|
+
const idField = test.idField ?? 'id';
|
|
639
|
+
const id = this.extractNestedValue(postJson, idField)
|
|
640
|
+
?? this.extractNestedValue(postJson, 'id');
|
|
641
|
+
if (id === undefined || id === null) {
|
|
642
|
+
return {
|
|
643
|
+
test,
|
|
644
|
+
status: 'fail',
|
|
645
|
+
statusCode: postStatus,
|
|
646
|
+
responseBody: postBody.slice(0, 2000),
|
|
647
|
+
errorMatch: `Could not extract ID from POST response using field "${idField}"`,
|
|
648
|
+
durationMs: Date.now() - start,
|
|
649
|
+
};
|
|
650
|
+
}
|
|
651
|
+
// Step 3: GET to verify persistence
|
|
652
|
+
const verifyPath = (test.verifyPath ?? '').replace(/\[id\]|:id|\{id\}/g, String(id));
|
|
653
|
+
const getController = new AbortController();
|
|
654
|
+
const getTimeoutId = setTimeout(() => getController.abort(), this.testTimeoutMs);
|
|
655
|
+
const getResponse = await fetch(`${baseUrl}${verifyPath}`, {
|
|
656
|
+
method: 'GET',
|
|
657
|
+
signal: getController.signal,
|
|
658
|
+
headers: { 'Accept': 'application/json' },
|
|
659
|
+
});
|
|
660
|
+
clearTimeout(getTimeoutId);
|
|
661
|
+
const getStatus = getResponse.status;
|
|
662
|
+
let getBody;
|
|
663
|
+
try {
|
|
664
|
+
getBody = await getResponse.text();
|
|
665
|
+
}
|
|
666
|
+
catch {
|
|
667
|
+
getBody = '';
|
|
668
|
+
}
|
|
669
|
+
if (getStatus === 404) {
|
|
670
|
+
return {
|
|
671
|
+
test,
|
|
672
|
+
status: 'fail',
|
|
673
|
+
statusCode: getStatus,
|
|
674
|
+
responseBody: `POST ${postStatus}: ${postBody.slice(0, 500)}\nGET ${getStatus}: ${getBody.slice(0, 500)}`,
|
|
675
|
+
errorMatch: 'Data did not persist — POST succeeded but GET returned 404',
|
|
676
|
+
durationMs: Date.now() - start,
|
|
677
|
+
};
|
|
678
|
+
}
|
|
679
|
+
if (getStatus >= 200 && getStatus < 300) {
|
|
680
|
+
return {
|
|
681
|
+
test,
|
|
682
|
+
status: 'pass',
|
|
683
|
+
statusCode: getStatus,
|
|
684
|
+
responseBody: `POST ${postStatus} → GET ${getStatus}: ${getBody.slice(0, 500)}`,
|
|
685
|
+
durationMs: Date.now() - start,
|
|
686
|
+
};
|
|
687
|
+
}
|
|
688
|
+
return {
|
|
689
|
+
test,
|
|
690
|
+
status: 'fail',
|
|
691
|
+
statusCode: getStatus,
|
|
692
|
+
responseBody: `POST ${postStatus}: ${postBody.slice(0, 500)}\nGET ${getStatus}: ${getBody.slice(0, 500)}`,
|
|
693
|
+
errorMatch: `GET verification returned unexpected status: ${getStatus}`,
|
|
694
|
+
durationMs: Date.now() - start,
|
|
695
|
+
};
|
|
696
|
+
}
|
|
697
|
+
catch (err) {
|
|
698
|
+
if (err instanceof Error && err.name === 'AbortError') {
|
|
699
|
+
return {
|
|
700
|
+
test,
|
|
701
|
+
status: 'timeout',
|
|
702
|
+
statusCode: null,
|
|
703
|
+
responseBody: '',
|
|
704
|
+
errorMatch: `Persistence test timed out after ${this.testTimeoutMs}ms`,
|
|
705
|
+
durationMs: Date.now() - start,
|
|
706
|
+
};
|
|
707
|
+
}
|
|
708
|
+
return {
|
|
709
|
+
test,
|
|
710
|
+
status: 'error',
|
|
711
|
+
statusCode: null,
|
|
712
|
+
responseBody: '',
|
|
713
|
+
errorMatch: err instanceof Error ? err.message : String(err),
|
|
714
|
+
durationMs: Date.now() - start,
|
|
715
|
+
};
|
|
716
|
+
}
|
|
717
|
+
}
|
|
718
|
+
/** Find a GET route that matches a POST route path with an ID segment appended. */
|
|
719
|
+
findCorrespondingGetRoute(postPath, allRoutes) {
|
|
720
|
+
const getRoutes = allRoutes.filter(r => r.method === 'GET');
|
|
721
|
+
// Try common dynamic segment patterns: [id], :id, {id}, [slug]
|
|
722
|
+
const candidates = [
|
|
723
|
+
`${postPath}/[id]`,
|
|
724
|
+
`${postPath}/:id`,
|
|
725
|
+
`${postPath}/{id}`,
|
|
726
|
+
`${postPath}/[slug]`,
|
|
727
|
+
];
|
|
728
|
+
for (const candidate of candidates) {
|
|
729
|
+
const match = getRoutes.find(r => r.path === candidate);
|
|
730
|
+
if (match)
|
|
731
|
+
return match.path;
|
|
732
|
+
}
|
|
733
|
+
// Also try: if POST is /api/foods, look for GET /api/foods/[foodId] etc.
|
|
734
|
+
const parts = postPath.split('/').filter(Boolean);
|
|
735
|
+
const resource = parts[parts.length - 1];
|
|
736
|
+
if (resource) {
|
|
737
|
+
const singular = resource.endsWith('s') ? resource.slice(0, -1) : resource;
|
|
738
|
+
const extraCandidates = [
|
|
739
|
+
`${postPath}/[${singular}Id]`,
|
|
740
|
+
`${postPath}/:${singular}Id`,
|
|
741
|
+
`${postPath}/[${singular}_id]`,
|
|
742
|
+
];
|
|
743
|
+
for (const candidate of extraCandidates) {
|
|
744
|
+
const match = getRoutes.find(r => r.path === candidate);
|
|
745
|
+
if (match)
|
|
746
|
+
return match.path;
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
return null;
|
|
750
|
+
}
|
|
751
|
+
/** Guess the ID field name from a route path (e.g., /api/foods → food.id). */
|
|
752
|
+
guessIdField(postPath) {
|
|
753
|
+
const parts = postPath.split('/').filter(Boolean);
|
|
754
|
+
const resource = parts[parts.length - 1];
|
|
755
|
+
if (!resource)
|
|
756
|
+
return 'id';
|
|
757
|
+
const singular = resource.endsWith('s') ? resource.slice(0, -1) : resource;
|
|
758
|
+
return `${singular}.id`;
|
|
759
|
+
}
|
|
760
|
+
/** Extract a nested value from an object using a dotted path (e.g., "food.id"). */
|
|
761
|
+
extractNestedValue(obj, dottedPath) {
|
|
762
|
+
if (obj === null || obj === undefined || typeof obj !== 'object')
|
|
763
|
+
return undefined;
|
|
764
|
+
const segments = dottedPath.split('.');
|
|
765
|
+
let current = obj;
|
|
766
|
+
for (const segment of segments) {
|
|
767
|
+
if (current === null || current === undefined || typeof current !== 'object') {
|
|
768
|
+
return undefined;
|
|
769
|
+
}
|
|
770
|
+
current = current[segment];
|
|
771
|
+
}
|
|
772
|
+
return current;
|
|
773
|
+
}
|
|
774
|
+
}
|
|
775
|
+
//# sourceMappingURL=functional-tester.js.map
|