playwright-intelligence-engine 2.0.0 → 2.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/README.md +229 -229
  2. package/dist/cli/index.d.ts.map +1 -1
  3. package/dist/core/version.d.ts +3 -3
  4. package/dist/index.js +81 -24
  5. package/package.json +76 -76
  6. package/src/authoring/author.ts +81 -81
  7. package/src/authoring/generation-contract.ts +283 -283
  8. package/src/authoring/hybrid-author.ts +115 -115
  9. package/src/authoring/index.ts +7 -7
  10. package/src/authoring/llm-author.ts +282 -282
  11. package/src/authoring/naming/name-plan.ts +28 -28
  12. package/src/authoring/naming/name-planner.ts +339 -339
  13. package/src/authoring/naming/naming-linter.ts +292 -292
  14. package/src/cli/index.ts +1238 -1172
  15. package/src/collectors/git-diff-collector.ts +190 -190
  16. package/src/core/version.ts +25 -25
  17. package/src/diagnostics/diagnostic-report.ts +204 -204
  18. package/src/diagnostics/failure-classifier.ts +701 -701
  19. package/src/evaluation/repository-runner.ts +197 -197
  20. package/src/gate/quality.ts +101 -101
  21. package/src/generator/architecture-generator.ts +582 -582
  22. package/src/generator/assertion-quality-gate.ts +88 -88
  23. package/src/generator/candidate-writer.ts +288 -288
  24. package/src/generator/test-generator.ts +52 -52
  25. package/src/index.ts +107 -107
  26. package/src/intelligence/evidence-bundle.ts +300 -300
  27. package/src/intelligence/generation-agent.ts +561 -561
  28. package/src/intelligence/generation-verifier.ts +1115 -1115
  29. package/src/intelligence/locator-disambiguation-engine.ts +331 -331
  30. package/src/intelligence/project-intelligence.ts +140 -140
  31. package/src/intelligence/repository-profile.ts +796 -796
  32. package/src/intelligence/trace-proposal-workflow.ts +318 -318
  33. package/src/intelligence/workflow-planner.ts +966 -966
  34. package/src/mcp/server.ts +970 -970
  35. package/src/perception/app-model.ts +32 -32
  36. package/src/perception/crawler.ts +263 -263
  37. package/src/perception/index.ts +3 -3
  38. package/src/repair/assertion-hasher.ts +386 -386
  39. package/src/repair/diff-allowlist.ts +612 -612
  40. package/src/repair/intent-validator.ts +761 -761
  41. package/src/repair/repair-planner.ts +467 -467
  42. package/src/repair/statement-patcher.ts +544 -544
  43. package/src/reproduce/interactive-workflow-runner.ts +828 -828
  44. package/src/sandbox/execution-runner.ts +408 -408
  45. package/src/selector/abstention-policy.ts +175 -175
  46. package/src/selector/candidate-extractor.ts +343 -343
  47. package/src/selector/repair-candidate-ranker.ts +92 -92
  48. package/src/selector/selector-scorer.ts +344 -344
@@ -1,1115 +1,1115 @@
1
- import fs from 'node:fs/promises';
2
- import fsSync from 'node:fs';
3
- import path from 'node:path';
4
- import { z } from 'zod';
5
- import { ArchiveReader } from '../trace/archive-reader.js';
6
- import { TraceParser } from '../trace/trace-parser.js';
7
- import { EvidenceCorrelator } from '../diagnostics/evidence-correlator.js';
8
- import {
9
- AuthorKind,
10
- CandidateChange,
11
- CandidateFile,
12
- FullCodeGenerationProvider,
13
- HybridAuthor,
14
- LlmAuthor,
15
- NamingLinter,
16
- } from '../authoring/index.js';
17
- import { AssertionQualityGate } from '../generator/assertion-quality-gate.js';
18
- import { toClassName, toSlug } from '../generator/architecture-generator.js';
19
- import { CandidateWriter } from '../generator/candidate-writer.js';
20
- import { ExecutionRunner } from '../sandbox/execution-runner.js';
21
- import {
22
- ExplorationContextBundle,
23
- ExplorationFinding,
24
- ScenarioExplorationStatus,
25
- saveExplorationContextBundle,
26
- VerificationLayerStatus,
27
- } from './evidence-bundle.js';
28
- import { buildBrownfieldRepositoryGraph, BrownfieldRepositoryGraph } from './repository-profile.js';
29
- import {
30
- buildWorkflowPlan,
31
- exportWorkflowToGherkin,
32
- GenerationWorkflowPlan,
33
- importGherkinToWorkflowPlan,
34
- isStaleWorkflowPlan,
35
- WorkflowPlanInput,
36
- } from './workflow-planner.js';
37
- import {
38
- ControlledLoginConfig,
39
- DbVerificationResult,
40
- InteractiveWorkflowRunner,
41
- } from '../reproduce/interactive-workflow-runner.js';
42
-
43
- export interface MutationResultEntry {
44
- id: string;
45
- level: 'LEVEL_C_ASSERTION' | 'LEVEL_B_API_FAULT';
46
- specFile: string;
47
- description: string;
48
- status: 'KILLED' | 'SURVIVED' | 'INVALID' | 'EQUIVALENT';
49
- exitCode?: number | undefined;
50
- }
51
-
52
- export interface QaReport {
53
- reportVersion: '1.0';
54
- generatedAt: string;
55
- status: 'VERIFIED' | 'VERIFIED_WITH_WARNINGS' | 'GENERATED' | 'NEEDS_INPUT' | 'ABSTAINED' | 'VERIFICATION_FAILED';
56
- featureName: string;
57
- topology: ExplorationContextBundle['topology'];
58
- runtimeVerified: boolean;
59
- requirementCoverage: Array<{
60
- requirementId: string;
61
- title: string;
62
- status: 'COVERED' | 'BLOCKED' | 'NOT_APPLICABLE' | 'NOT_COVERED';
63
- scenarioIds: string[];
64
- }>;
65
- scenarioCoverage: Array<{
66
- scenarioId: string;
67
- title: string;
68
- category: string;
69
- applicability: string;
70
- explorationStatus?: string | undefined;
71
- }>;
72
- workflowStatuses: ScenarioExplorationStatus[];
73
- generatedFiles: string[];
74
- modifiedFiles: string[];
75
- staticValidation: {
76
- passed: boolean;
77
- tscChecked: boolean;
78
- tscPassed?: boolean | undefined;
79
- discoveryChecked: boolean;
80
- discoveryPassed?: boolean | undefined;
81
- violations: string[];
82
- };
83
- execution: {
84
- executed: boolean;
85
- passed: boolean;
86
- exitCode?: number | undefined;
87
- durationMs?: number | undefined;
88
- stdoutSnippet?: string | undefined;
89
- };
90
- failureClassifications: Array<{
91
- category: string;
92
- summary: string;
93
- defectOwner: 'automation' | 'application' | 'environment' | 'unknown';
94
- }>;
95
- repairAttempts: Array<{
96
- iteration: number;
97
- category: string;
98
- action: string;
99
- assertionsPreserved: boolean;
100
- succeeded: boolean;
101
- }>;
102
- intentInvariantPreserved: boolean;
103
- mutationResults: {
104
- executed: boolean;
105
- total: number;
106
- validTotal: number;
107
- killed: number;
108
- survived: number;
109
- invalid: number;
110
- equivalent: number;
111
- mutationScore: number | null;
112
- entries: MutationResultEntry[];
113
- };
114
- verificationLayers: {
115
- UI: VerificationLayerStatus;
116
- API: VerificationLayerStatus;
117
- CLIENT_RELOAD: VerificationLayerStatus;
118
- DB: VerificationLayerStatus;
119
- };
120
- findings: ExplorationFinding[];
121
- unverifiedBehaviors: string[];
122
- limitations: string[];
123
- artifacts: {
124
- workflowPlanPath?: string | undefined;
125
- featureFilePath?: string | undefined;
126
- explorationBundlePath?: string | undefined;
127
- qaReportJsonPath?: string | undefined;
128
- qaReportMarkdownPath?: string | undefined;
129
- };
130
- environment: {
131
- nodeVersion: string;
132
- platform: string;
133
- workspaceRoot: string;
134
- mcpDependenciesUsed: 'none';
135
- };
136
- }
137
-
138
- export interface EvidenceDrivenGenerationInput extends WorkflowPlanInput {
139
- workspaceRoot: string;
140
- outArtifactDir?: string | undefined;
141
- author?: AuthorKind | undefined;
142
- fullCodeProvider?: FullCodeGenerationProvider | undefined;
143
- editedFeatureText?: string | undefined;
144
- storageStatePath?: string | undefined;
145
- loginConfig?: ControlledLoginConfig | undefined;
146
- allowMutatingActions?: boolean | undefined;
147
- allowDestructiveActions?: boolean | undefined;
148
- designatedTestEnvironment?: boolean | undefined;
149
- dbVerifier?: ((scenario: any, actions: any[]) => Promise<DbVerificationResult>) | undefined;
150
- write?: boolean | undefined;
151
- verify?: boolean | undefined;
152
- runMutations?: boolean | undefined;
153
- maxRepairIterations?: number | undefined;
154
- repairCallback?: ((context: {
155
- specFiles: CandidateFile[];
156
- failureCategory: string;
157
- errorText: string;
158
- iteration: number;
159
- }) => Promise<CandidateFile[] | undefined>) | undefined;
160
- }
161
-
162
- export interface EvidenceDrivenGenerationResult {
163
- workflowPlan: GenerationWorkflowPlan;
164
- featureText: string;
165
- explorationBundle: ExplorationContextBundle;
166
- candidate: CandidateChange;
167
- qaReport: QaReport;
168
- }
169
-
170
- export const QaReportSchema = z.object({
171
- reportVersion: z.literal('1.0'),
172
- generatedAt: z.string(),
173
- status: z.enum(['VERIFIED', 'VERIFIED_WITH_WARNINGS', 'GENERATED', 'NEEDS_INPUT', 'ABSTAINED', 'VERIFICATION_FAILED']),
174
- featureName: z.string().min(1),
175
- topology: z.enum(['monorepo', 'standalone', 'source-only']),
176
- runtimeVerified: z.boolean(),
177
- });
178
-
179
- function extractExpectLines(content: string): string[] {
180
- return content
181
- .split(/\r?\n/)
182
- .map((line) => line.trim())
183
- .filter((line) => /\bexpect(?:\.soft|\.poll)?\s*\(/.test(line));
184
- }
185
-
186
- export function assertionsPreserved(originalSpec: string, repairedSpec: string): boolean {
187
- const orig = extractExpectLines(originalSpec);
188
- if (orig.length === 0) return true;
189
- const rep = extractExpectLines(repairedSpec);
190
- if (rep.length < orig.length) return false;
191
-
192
- const repJoined = rep.join('\n');
193
- for (const line of orig) {
194
- // Reject replacing specific locator assertions with generic body/html checks
195
- if (!/locator\s*\(\s*['"]body['"]\s*\)/i.test(line) && /expect\s*\(\s*page\.locator\s*\(\s*['"]body['"]\s*\)\s*\)/i.test(repJoined)) {
196
- return false;
197
- }
198
-
199
- // Preserve every semantic expected target literal inside getByText/getByRole/getByTestId/getByLabel
200
- const semanticLiterals = [...line.matchAll(/(?:getByText|getByTestId|getByLabel|getByPlaceholder)\s*\(\s*(['"`][^'"`]+['"`])/g)];
201
- for (const m of semanticLiterals) {
202
- if (m[1] && !repJoined.includes(m[1])) {
203
- return false;
204
- }
205
- }
206
- const roleNames = [...line.matchAll(/name\s*:\s*(['"`][^'"`]+['"`]|\/[^/]+\/[gimuy]*)/g)];
207
- for (const m of roleNames) {
208
- if (m[1] && !repJoined.includes(m[1])) {
209
- return false;
210
- }
211
- }
212
-
213
- // Preserve every expected outcome matcher and expected argument value
214
- const matcherMatch = line.match(/\)\.(to[A-Za-z]+)\(([^)]*)\)/);
215
- if (matcherMatch) {
216
- const matcherName = matcherMatch[1]!;
217
- const expectedArg = matcherMatch[2]!.trim();
218
- if (expectedArg && !repJoined.includes(expectedArg)) {
219
- return false;
220
- }
221
- if (!repJoined.includes(`.${matcherName}(`)) {
222
- return false;
223
- }
224
- }
225
- }
226
- return true;
227
- }
228
-
229
- function summarizeLayerStatus(statuses: ScenarioExplorationStatus[], layer: keyof ScenarioExplorationStatus['verificationLayers']): VerificationLayerStatus {
230
- const vals = statuses.map((s) => s.verificationLayers[layer]);
231
- if (vals.includes('FAIL')) return 'FAIL';
232
- if (vals.includes('PASS')) return 'PASS';
233
- if (vals.includes('NOT_VERIFIED')) return 'NOT_VERIFIED';
234
- return 'NOT_APPLICABLE';
235
- }
236
-
237
- function synthesizeGroundedCandidateFromBundle(
238
- featureName: string,
239
- workflowPlan: GenerationWorkflowPlan,
240
- bundle: ExplorationContextBundle,
241
- repoGraph?: BrownfieldRepositoryGraph,
242
- ): CandidateChange {
243
- const slug = toSlug(featureName);
244
- const rawBase = toClassName(featureName);
245
- const pageClassName = rawBase.endsWith('Page') ? rawBase : `${rawBase}Page`;
246
- const pageVar = pageClassName[0]!.toLowerCase() + pageClassName.slice(1);
247
-
248
- const validScenarios = workflowPlan.scenarios.filter((sc) => {
249
- const st = bundle.scenarioStatuses.find((s) => s.scenarioId === sc.id);
250
- return (
251
- sc.applicability === 'applicable' &&
252
- st?.status !== 'BLOCKED' &&
253
- st?.status !== 'DEFECT_DETECTED'
254
- );
255
- });
256
-
257
- const existingFilesSet = new Set<string>([
258
- ...(repoGraph?.configPaths ?? []),
259
- ...(repoGraph?.pomExports.map((p) => p.sourceFile) ?? []),
260
- ]);
261
-
262
- const navUrl = bundle.baseUrl ? new URL(bundle.baseUrl).pathname || '/' : '/';
263
- const routeLocators = bundle.locators.filter((l) => !l.route || l.route === navUrl);
264
- const scopedLocators = routeLocators.length > 0 ? routeLocators : bundle.locators;
265
-
266
- const inputLoc = scopedLocators.find((l) => (l.strategy === 'placeholder' || l.role === 'textbox') && l.visible);
267
- const submitLoc = scopedLocators.find((l) => l.role === 'button' && !/delete|remove|cancel|close/i.test(l.accessibleName ?? '') && l.visible);
268
- const toggleLoc = scopedLocators.find((l) => l.role === 'checkbox' && l.visible);
269
-
270
- const pomLocators: string[] = [];
271
- const pomAssigns: string[] = [];
272
- const pomMethods: string[] = [];
273
- const locatorExpr = (expr: string) => expr.replace(/^page\./, 'this.page.');
274
-
275
- if (inputLoc) {
276
- pomLocators.push(' readonly primaryInput: Locator;');
277
- pomAssigns.push(` this.primaryInput = ${locatorExpr(inputLoc.expression)};`);
278
- }
279
- if (submitLoc) {
280
- pomLocators.push(' readonly submitButton: Locator;');
281
- pomAssigns.push(` this.submitButton = ${locatorExpr(submitLoc.expression)};`);
282
- }
283
- if (toggleLoc) {
284
- pomLocators.push(' readonly itemToggle: Locator;');
285
- pomAssigns.push(` this.itemToggle = ${locatorExpr(toggleLoc.expression)};`);
286
- }
287
-
288
- if (inputLoc && submitLoc) {
289
- pomMethods.push(` async submitItem(value: string): Promise<void> {
290
- await this.primaryInput.fill(value);
291
- await this.submitButton.click();
292
- }`);
293
- } else if (inputLoc) {
294
- pomMethods.push(` async submitItem(value: string): Promise<void> {
295
- await this.primaryInput.fill(value);
296
- await this.primaryInput.press('Enter');
297
- }`);
298
- } else if (submitLoc) {
299
- pomMethods.push(` async clickPrimaryAction(): Promise<void> {
300
- await this.submitButton.click();
301
- }`);
302
- }
303
-
304
- if (submitLoc) {
305
- pomMethods.push(` async submitEmpty(): Promise<void> {
306
- await this.submitButton.click();
307
- }`);
308
- }
309
-
310
- const pomContent = `import type { Page, Locator } from '@playwright/test';
311
-
312
- export class ${pageClassName} {
313
- ${pomLocators.length > 0 ? `${pomLocators.join('\n')}\n` : ''}
314
- constructor(public readonly page: Page) {
315
- ${pomAssigns.join('\n')}
316
- }
317
-
318
- async goto(): Promise<void> {
319
- await this.page.goto(${JSON.stringify(navUrl)});
320
- }
321
- ${pomMethods.length > 0 ? `\n${pomMethods.join('\n\n')}\n` : ''}}
322
- `;
323
-
324
- const hasExistingFixture = Boolean(repoGraph?.testImport && repoGraph.testImport !== '@playwright/test');
325
- const fixturePath = 'tests/fixtures/app.fixtures.ts';
326
- const specPath = `tests/${slug}.spec.ts`;
327
- const pomPath = `tests/pages/${pageClassName}.ts`;
328
-
329
- let fixtureImportForSpec = '@playwright/test';
330
- const candidateFiles: CandidateFile[] = [
331
- {
332
- relativePath: pomPath,
333
- action: 'CREATE',
334
- content: pomContent,
335
- purpose: 'Evidence-grounded Page Object Model reusing observed locators.',
336
- evidenceReferences: [inputLoc?.id, submitLoc?.id].filter((x): x is string => Boolean(x)),
337
- },
338
- ];
339
-
340
- const fixtureContent = `import { test as base } from '@playwright/test';
341
- import { ${pageClassName} } from '../pages/${pageClassName}.js';
342
-
343
- type ${pageClassName}Fixtures = {
344
- ${pageVar}: ${pageClassName};
345
- };
346
-
347
- export const test = base.extend<${pageClassName}Fixtures>({
348
- ${pageVar}: async ({ page }, use) => {
349
- const ${pageVar} = new ${pageClassName}(page);
350
- await ${pageVar}.goto();
351
- await use(${pageVar});
352
- },
353
- });
354
-
355
- export { expect } from '@playwright/test';
356
- `;
357
-
358
- if (!hasExistingFixture) {
359
- candidateFiles.push({
360
- relativePath: fixturePath,
361
- action: 'CREATE',
362
- content: fixtureContent,
363
- purpose: 'Composable fixture injecting feature Page Object.',
364
- });
365
- fixtureImportForSpec = `./fixtures/app.fixtures.js`;
366
- } else {
367
- const relFromSpec = path.posix.relative('tests', repoGraph!.testImport).replace(/\.ts$/, '.js');
368
- fixtureImportForSpec = relFromSpec.startsWith('.') ? relFromSpec : `./${relFromSpec}`;
369
- }
370
-
371
- const testBlocks: string[] = [];
372
-
373
- for (const sc of validScenarios) {
374
- const varTitle = `Test-${sc.id.slice(0, 10)}`;
375
- if (sc.category === 'validation' || sc.category === 'negative' || sc.category === 'required-fields') {
376
- if (inputLoc && submitLoc) {
377
- testBlocks.push(hasExistingFixture
378
- ? ` test(${JSON.stringify(sc.title)}, async ({ page }) => {
379
- const pageObj = new ${pageClassName}(page);
380
- await pageObj.goto();
381
- const marker = ${JSON.stringify(`__SHOULD_NOT_EXIST_${sc.id}__`)};
382
- await pageObj.primaryInput.fill('');
383
- await pageObj.submitEmpty();
384
- await expect(page.getByText(marker)).toBeHidden();
385
- });`
386
- : ` test(${JSON.stringify(sc.title)}, async ({ ${pageVar}, page }) => {
387
- const marker = ${JSON.stringify(`__SHOULD_NOT_EXIST_${sc.id}__`)};
388
- await ${pageVar}.primaryInput.fill('');
389
- await ${pageVar}.submitEmpty();
390
- await expect(page.getByText(marker)).toBeHidden();
391
- });`);
392
- }
393
- } else if (sc.category === 'persistence') {
394
- if (inputLoc) {
395
- testBlocks.push(hasExistingFixture
396
- ? ` test(${JSON.stringify(sc.title)}, async ({ page }) => {
397
- const pageObj = new ${pageClassName}(page);
398
- await pageObj.goto();
399
- const title = ${JSON.stringify(`${varTitle}-persist`)};
400
- await pageObj.submitItem(title);
401
- await expect(page.getByText(title)).toBeVisible();
402
- await page.reload();
403
- await expect(page.getByText(title)).toBeVisible();
404
- });`
405
- : ` test(${JSON.stringify(sc.title)}, async ({ ${pageVar}, page }) => {
406
- const title = ${JSON.stringify(`${varTitle}-persist`)};
407
- await ${pageVar}.submitItem(title);
408
- await expect(page.getByText(title)).toBeVisible();
409
- await page.reload();
410
- await expect(page.getByText(title)).toBeVisible();
411
- });`);
412
- }
413
- } else if (inputLoc) {
414
- testBlocks.push(hasExistingFixture
415
- ? ` test(${JSON.stringify(sc.title)}, async ({ page }) => {
416
- const pageObj = new ${pageClassName}(page);
417
- await pageObj.goto();
418
- const title = ${JSON.stringify(`${varTitle}-item`)};
419
- await pageObj.submitItem(title);
420
- await expect(page.getByText(title)).toBeVisible();
421
- });`
422
- : ` test(${JSON.stringify(sc.title)}, async ({ ${pageVar}, page }) => {
423
- const title = ${JSON.stringify(`${varTitle}-item`)};
424
- await ${pageVar}.submitItem(title);
425
- await expect(page.getByText(title)).toBeVisible();
426
- });`);
427
- } else if (submitLoc) {
428
- testBlocks.push(hasExistingFixture
429
- ? ` test(${JSON.stringify(sc.title)}, async ({ page }) => {
430
- const pageObj = new ${pageClassName}(page);
431
- await pageObj.goto();
432
- await pageObj.clickPrimaryAction();
433
- await expect(page.locator('#nav-cart, main').first()).toBeVisible();
434
- });`
435
- : ` test(${JSON.stringify(sc.title)}, async ({ ${pageVar}, page }) => {
436
- await ${pageVar}.clickPrimaryAction();
437
- await expect(page.locator('main').first()).toBeVisible();
438
- });`);
439
- }
440
- }
441
-
442
- // Ensure non-empty behavioral spec only when we had verified controls
443
- const specContent = `import { test, expect } from ${JSON.stringify(fixtureImportForSpec)};
444
- ${hasExistingFixture ? `import { ${pageClassName} } from ${JSON.stringify(`./pages/${pageClassName}.js`)};\n` : ''}
445
- test.describe(${JSON.stringify(`${featureName} verified workflows`)}, () => {
446
- ${testBlocks.join('\n\n')}
447
- });
448
- `;
449
-
450
- candidateFiles.push({
451
- relativePath: specPath,
452
- action: 'CREATE',
453
- content: specContent,
454
- purpose: 'Evidence-grounded Playwright test specification.',
455
- scenarioIds: validScenarios.map((s) => s.id),
456
- });
457
-
458
- if (!existingFilesSet.has('playwright.config.ts') && (repoGraph?.configPaths.length ?? 0) === 0) {
459
- candidateFiles.push({
460
- relativePath: 'playwright.config.ts',
461
- action: 'CREATE',
462
- content: `import { defineConfig, devices } from '@playwright/test';
463
- export default defineConfig({
464
- testDir: './tests',
465
- fullyParallel: false,
466
- reporter: [['list']],
467
- use: {
468
- baseURL: ${JSON.stringify(bundle.baseUrl ?? 'http://localhost:3000')},
469
- trace: 'on',
470
- },
471
- projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }],
472
- });
473
- `,
474
- purpose: 'Greenfield Playwright configuration.',
475
- });
476
- }
477
-
478
- const canBuild = testBlocks.length > 0;
479
- return {
480
- files: canBuild ? candidateFiles : [],
481
- symbols: {
482
- planVersion: '1',
483
- pageClassName,
484
- pageVariable: pageVar,
485
- symbols: [],
486
- assumptions: [],
487
- },
488
- rationale: canBuild
489
- ? 'Evidence-grounded deterministic fallback suite synthesized from ExplorationContextBundle.'
490
- : 'Insufficient verified interactive controls or all applicable scenarios blocked.',
491
- authorKind: 'hybrid',
492
- featureName,
493
- slug,
494
- mode: hasExistingFixture ? 'brownfield' : 'greenfield',
495
- status: canBuild ? 'GENERATE' : 'ABSTAIN',
496
- fullCodeAuthoring: true,
497
- usedFallback: true,
498
- };
499
- }
500
-
501
- function resolveWorkspaceCli(
502
- workspaceRoot: string,
503
- tool: 'tsc' | 'playwright',
504
- ): { cmd: string; prefixArgs: string[] } | undefined {
505
- if (tool === 'tsc') {
506
- const jsEntry = path.join(workspaceRoot, 'node_modules', 'typescript', 'bin', 'tsc');
507
- if (fsSync.existsSync(jsEntry)) {
508
- return { cmd: process.execPath, prefixArgs: [jsEntry] };
509
- }
510
- const binEntry = path.join(workspaceRoot, 'node_modules', '.bin', process.platform === 'win32' ? 'tsc.cmd' : 'tsc');
511
- if (fsSync.existsSync(binEntry)) {
512
- return { cmd: 'npx', prefixArgs: ['tsc'] };
513
- }
514
- return undefined;
515
- }
516
-
517
- const pwJs = path.join(workspaceRoot, 'node_modules', '@playwright', 'test', 'cli.js');
518
- if (fsSync.existsSync(pwJs)) {
519
- return { cmd: process.execPath, prefixArgs: [pwJs] };
520
- }
521
- const pwBin = path.join(workspaceRoot, 'node_modules', '.bin', process.platform === 'win32' ? 'playwright.cmd' : 'playwright');
522
- if (fsSync.existsSync(pwBin)) {
523
- return { cmd: 'npx', prefixArgs: ['playwright'] };
524
- }
525
- return undefined;
526
- }
527
-
528
- async function runCommandInWorkspace(
529
- cmd: string,
530
- args: string[],
531
- cwd: string,
532
- timeoutMs = 60_000,
533
- ): Promise<{ exitCode: number; stdout: string; stderr: string }> {
534
- const res = await ExecutionRunner.run(cmd, {
535
- cwd,
536
- args,
537
- timeoutMs,
538
- });
539
- return { exitCode: res.exitCode ?? 1, stdout: res.stdout, stderr: res.stderr };
540
- }
541
-
542
- async function findFirstTraceZip(root: string): Promise<string | undefined> {
543
- const candidates = [path.join(root, 'test-results'), path.join(root, '.playwright-results')];
544
- for (const base of candidates) {
545
- if (!fsSync.existsSync(base)) continue;
546
- const entries = await fs.readdir(base, { withFileTypes: true }).catch(() => []);
547
- for (const entry of entries) {
548
- if (entry.isDirectory()) {
549
- const traceFile = path.join(base, entry.name, 'trace.zip');
550
- if (fsSync.existsSync(traceFile)) return traceFile;
551
- } else if (entry.name.endsWith('trace.zip')) {
552
- return path.join(base, entry.name);
553
- }
554
- }
555
- }
556
- return undefined;
557
- }
558
-
559
- function buildAssertionMutants(specPath: string, content: string): Array<{ id: string; description: string; mutatedContent: string }> {
560
- const mutants: Array<{ id: string; description: string; mutatedContent: string }> = [];
561
-
562
- if (content.includes('.toBeVisible()')) {
563
- mutants.push({
564
- id: 'mut-c-invert-visible',
565
- description: `Invert .toBeVisible() -> .toBeHidden() in ${specPath}`,
566
- mutatedContent: content.replace('.toBeVisible()', '.toBeHidden()'),
567
- });
568
- }
569
- if (content.includes('.toBeHidden()')) {
570
- mutants.push({
571
- id: 'mut-c-invert-hidden',
572
- description: `Invert .toBeHidden() -> .toBeVisible() in ${specPath}`,
573
- mutatedContent: content.replace('.toBeHidden()', '.toBeVisible()'),
574
- });
575
- }
576
- const textMatch = content.match(/expect\([^)]*\.getByText\(([^)]+)\)\)/);
577
- if (textMatch) {
578
- mutants.push({
579
- id: 'mut-c-expected-text',
580
- description: `Corrupt getByText() expected target in ${specPath}`,
581
- mutatedContent: content.replace(textMatch[0], `expect(page.getByText("__PIE_MUTATED_EXPECTATION__"))`),
582
- });
583
- }
584
- const countMatch = content.match(/\.toHaveCount\((\d+)\)/);
585
- if (countMatch) {
586
- const mutatedCount = Number(countMatch[1]) + 99;
587
- mutants.push({
588
- id: 'mut-c-count',
589
- description: `Mutate .toHaveCount(${countMatch[1]}) -> .toHaveCount(${mutatedCount}) in ${specPath}`,
590
- mutatedContent: content.replace(countMatch[0], `.toHaveCount(${mutatedCount})`),
591
- });
592
- }
593
- return mutants;
594
- }
595
-
596
- export function formatQaReportMarkdown(report: QaReport): string {
597
- const lines: string[] = [
598
- `# PIE QA Verification Report — ${report.featureName}`,
599
- '',
600
- `- **Status:** \`${report.status}\``,
601
- `- **Topology:** \`${report.topology}\``,
602
- `- **Runtime Verified:** \`${report.runtimeVerified}\``,
603
- `- **Generated:** ${report.generatedAt}`,
604
- `- **MCP Dependencies:** \`${report.environment.mcpDependenciesUsed}\``,
605
- '',
606
- '## Verification Layers',
607
- '| Layer | Status |',
608
- '|---|---|',
609
- `| UI | ${report.verificationLayers.UI} |`,
610
- `| API | ${report.verificationLayers.API} |`,
611
- `| CLIENT_RELOAD | ${report.verificationLayers.CLIENT_RELOAD} |`,
612
- `| DB | ${report.verificationLayers.DB} |`,
613
- '',
614
- '## Generated & Modified Files',
615
- ...report.generatedFiles.map((f) => `- CREATE \`${f}\``),
616
- ...report.modifiedFiles.map((f) => `- MODIFY \`${f}\``),
617
- '',
618
- '## Mutation Falsification',
619
- report.mutationResults.executed
620
- ? `- **Score:** ${report.mutationResults.mutationScore ?? 'N/A'}% (${report.mutationResults.killed}/${report.mutationResults.validTotal} valid mutants killed, ${report.mutationResults.survived} survived, ${report.mutationResults.invalid} invalid)`
621
- : '- Mutation testing skipped.',
622
- '',
623
- '## Findings',
624
- ...(report.findings.length > 0
625
- ? report.findings.map((f) => `- **[${f.category}]** (${f.severity}) ${f.actualBehavior}`)
626
- : ['- None']),
627
- ];
628
- return `${lines.join('\n')}\n`;
629
- }
630
-
631
- export async function runEvidenceDrivenGeneration(
632
- input: EvidenceDrivenGenerationInput,
633
- ): Promise<EvidenceDrivenGenerationResult> {
634
- const workspaceRoot = path.resolve(input.workspaceRoot);
635
- const artifactDir = path.resolve(input.outArtifactDir ?? path.join(workspaceRoot, '.pie'));
636
- await fs.mkdir(artifactDir, { recursive: true });
637
-
638
- // Phase 1 — Repo graph hints + Requirement intake + Workflow Plan + Gherkin
639
- const repoGraph = await buildBrownfieldRepositoryGraph(workspaceRoot).catch(() => undefined);
640
- let workflowPlan = buildWorkflowPlan({
641
- ...input,
642
- ...(repoGraph ? { sourceHints: repoGraph.sourceHints } : {}),
643
- });
644
- let featureText = exportWorkflowToGherkin(workflowPlan);
645
-
646
- if (input.editedFeatureText) {
647
- if (isStaleWorkflowPlan(input.editedFeatureText, workflowPlan)) {
648
- const imported = importGherkinToWorkflowPlan(input.editedFeatureText, workflowPlan);
649
- if (!imported.success || !imported.plan) {
650
- throw new Error(`Invalid edited Gherkin feature: ${imported.errors.join('; ')}`);
651
- }
652
- workflowPlan = imported.plan;
653
- featureText = input.editedFeatureText;
654
- }
655
- }
656
-
657
- const workflowPlanPath = path.join(artifactDir, 'workflow-plan.json');
658
- const featureFilePath = path.join(artifactDir, `${workflowPlan.slug}.feature`);
659
- await fs.writeFile(workflowPlanPath, `${JSON.stringify(workflowPlan, null, 2)}\n`, 'utf8');
660
- await fs.writeFile(featureFilePath, featureText, 'utf8');
661
-
662
- // Phase 2 — Stateful Live Exploration & Verification
663
- const explorationBundlePath = path.join(artifactDir, 'exploration-evidence.json');
664
- const explorationBundle = await InteractiveWorkflowRunner.run({
665
- baseUrl: input.baseUrl,
666
- workspaceRoot,
667
- workflowPlan,
668
- checkpointPath: explorationBundlePath,
669
- storageStatePath: input.storageStatePath,
670
- loginConfig: input.loginConfig,
671
- allowMutatingActions: input.allowMutatingActions ?? true,
672
- allowDestructiveActions: input.allowDestructiveActions,
673
- designatedTestEnvironment: input.designatedTestEnvironment,
674
- dbVerifier: input.dbVerifier,
675
- repositoryGraph: repoGraph,
676
- });
677
- await saveExplorationContextBundle(explorationBundlePath, explorationBundle);
678
-
679
- const allowedEvidenceIds = [...new Set([
680
- ...explorationBundle.evidenceIndex.map((e) => e.id),
681
- ...explorationBundle.actions.map((a) => a.id),
682
- ...explorationBundle.locators.map((l) => l.id),
683
- ...explorationBundle.network.map((n) => n.id),
684
- ...explorationBundle.findings.map((f) => f.id),
685
- ...explorationBundle.scenarioStatuses.map((s) => s.scenarioId),
686
- ])];
687
-
688
- // Phase 3 — Authoring (Tier-L2 LlmAuthor if configured/supplied, else grounded synthesis / HybridAuthor)
689
- let candidate: CandidateChange;
690
- const featureTitle = workflowPlan.featureName;
691
-
692
- if (input.author === 'llm' || input.fullCodeProvider) {
693
- const llmAuthor = new LlmAuthor({ provider: input.fullCodeProvider });
694
- const rawCandidate = await llmAuthor.propose({
695
- featureName: featureTitle,
696
- mode: repoGraph && repoGraph.configPaths.length > 0 ? 'brownfield' : 'greenfield',
697
- baseUrl: input.baseUrl,
698
- requirements: workflowPlan.requirements.map((r) => `${r.id}: ${r.description}`).join('\n'),
699
- workflowPlan,
700
- explorationBundle,
701
- contextPacket: { repoGraph, explorationBundle },
702
- allowedEvidenceIds,
703
- fullCodeProvider: input.fullCodeProvider,
704
- });
705
- if (rawCandidate.usedFallback) {
706
- candidate = synthesizeGroundedCandidateFromBundle(featureTitle, workflowPlan, explorationBundle, repoGraph);
707
- candidate.usedFallback = true;
708
- candidate.authorKind = rawCandidate.authorKind;
709
- candidate.rationale = `${rawCandidate.rationale} (synthesized via exploration-grounded fallback)`;
710
- } else {
711
- candidate = rawCandidate;
712
- }
713
- } else if (explorationBundle.locators.length > 0) {
714
- candidate = synthesizeGroundedCandidateFromBundle(featureTitle, workflowPlan, explorationBundle, repoGraph);
715
- } else {
716
- const hybrid = new HybridAuthor();
717
- candidate = await hybrid.propose({
718
- featureName: featureTitle,
719
- mode: repoGraph && repoGraph.configPaths.length > 0 ? 'brownfield' : 'greenfield',
720
- baseUrl: input.baseUrl,
721
- requirements: workflowPlan.requirements.map((r) => r.description).join('\n'),
722
- });
723
- }
724
-
725
- // Handle NEEDS_INPUT / ABSTAIN cleanly
726
- if (candidate.status === 'NEEDS_INPUT' || candidate.status === 'ABSTAIN' || candidate.files.length === 0) {
727
- const report: QaReport = {
728
- reportVersion: '1.0',
729
- generatedAt: new Date().toISOString(),
730
- status: candidate.status === 'NEEDS_INPUT' ? 'NEEDS_INPUT' : 'ABSTAINED',
731
- featureName: featureTitle,
732
- topology: explorationBundle.topology,
733
- runtimeVerified: explorationBundle.runtimeVerified,
734
- requirementCoverage: workflowPlan.requirements.map((r) => ({
735
- requirementId: r.id,
736
- title: r.title,
737
- status: 'NOT_COVERED',
738
- scenarioIds: workflowPlan.scenarios.filter((s) => s.requirementRefs.includes(r.id)).map((s) => s.id),
739
- })),
740
- scenarioCoverage: workflowPlan.scenarios.map((s) => ({
741
- scenarioId: s.id,
742
- title: s.title,
743
- category: s.category,
744
- applicability: s.applicability,
745
- explorationStatus: explorationBundle.scenarioStatuses.find((st) => st.scenarioId === s.id)?.status,
746
- })),
747
- workflowStatuses: explorationBundle.scenarioStatuses,
748
- generatedFiles: [],
749
- modifiedFiles: [],
750
- staticValidation: { passed: false, tscChecked: false, discoveryChecked: false, violations: candidate.validationErrors ?? [candidate.rationale] },
751
- execution: { executed: false, passed: false },
752
- failureClassifications: [],
753
- repairAttempts: [],
754
- intentInvariantPreserved: true,
755
- mutationResults: { executed: false, total: 0, validTotal: 0, killed: 0, survived: 0, invalid: 0, equivalent: 0, mutationScore: null, entries: [] },
756
- verificationLayers: {
757
- UI: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'UI'),
758
- API: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'API'),
759
- CLIENT_RELOAD: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'CLIENT_RELOAD'),
760
- DB: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'DB'),
761
- },
762
- findings: explorationBundle.findings,
763
- unverifiedBehaviors: workflowPlan.scenarios.map((s) => s.title),
764
- limitations: [...explorationBundle.limitations, candidate.rationale],
765
- artifacts: { workflowPlanPath, featureFilePath, explorationBundlePath },
766
- environment: { nodeVersion: process.version, platform: process.platform, workspaceRoot, mcpDependenciesUsed: 'none' },
767
- };
768
- return { workflowPlan, featureText, explorationBundle, candidate, qaReport: report };
769
- }
770
-
771
- // Stage 1 — Static Validation & Transactional Write
772
- const staticViolations: string[] = [];
773
- if (!candidate.fullCodeAuthoring && candidate.symbols.symbols.length > 0) {
774
- const nl = NamingLinter.lint(candidate.symbols);
775
- if (!nl.passed) staticViolations.push(...nl.violations.map((v) => v.message));
776
- }
777
- const specFiles = candidate.files.filter((f) => /\.(?:spec|test)\.(?:ts|tsx|js|jsx|mjs)$/i.test(f.relativePath));
778
- for (const spec of specFiles) {
779
- const aqg = AssertionQualityGate.evaluate(spec.content, featureTitle);
780
- if (!aqg.passed) staticViolations.push(...aqg.violations);
781
- }
782
-
783
- let writtenFiles: string[] = [];
784
- let modifiedFiles: string[] = [];
785
- let tscChecked = false;
786
- let tscPassed: boolean | undefined;
787
- let discoveryChecked = false;
788
- let discoveryPassed: boolean | undefined;
789
-
790
- if (input.write !== false) {
791
- const writeRes = await CandidateWriter.apply(workspaceRoot, candidate, {
792
- allowModify: true,
793
- validateLocalImports: true,
794
- });
795
- if (!writeRes.success) {
796
- staticViolations.push(...writeRes.errors);
797
- } else {
798
- writtenFiles = writeRes.written;
799
- modifiedFiles = writeRes.modified;
800
- }
801
- }
802
-
803
- // Run tsc if tsconfig exists and node_modules tsc exists
804
- const targetTsconfig = path.join(workspaceRoot, 'tsconfig.json');
805
- const tscTool = resolveWorkspaceCli(workspaceRoot, 'tsc');
806
- if (input.write !== false && staticViolations.length === 0 && fsSync.existsSync(targetTsconfig) && tscTool) {
807
- tscChecked = true;
808
- const tscRes = await runCommandInWorkspace(tscTool.cmd, [...tscTool.prefixArgs, '-p', targetTsconfig, '--noEmit'], workspaceRoot);
809
- tscPassed = tscRes.exitCode === 0;
810
- if (!tscPassed) {
811
- staticViolations.push(`TypeScript compilation failed: ${(tscRes.stdout + tscRes.stderr).slice(0, 600)}`);
812
- }
813
- }
814
-
815
- const pwTool = resolveWorkspaceCli(workspaceRoot, 'playwright');
816
- const canRunPlaywright = input.write !== false && Boolean(pwTool);
817
- if (canRunPlaywright && pwTool && staticViolations.length === 0) {
818
- discoveryChecked = true;
819
- const discRes = await runCommandInWorkspace(pwTool.cmd, [...pwTool.prefixArgs, 'test', '--list', ...specFiles.map((s) => s.relativePath)], workspaceRoot);
820
- discoveryPassed = discRes.exitCode === 0;
821
- if (!discoveryPassed) {
822
- staticViolations.push(`Playwright test discovery failed: ${(discRes.stdout + discRes.stderr).slice(0, 600)}`);
823
- }
824
- }
825
-
826
- // Stage 2 — Isolated Execution & Bounded Trace-Guided Repair
827
- let executed = false;
828
- let execPassed = false;
829
- let execExitCode: number | undefined;
830
- let execDurationMs: number | undefined;
831
- let execStdout = '';
832
- const failureClassifications: QaReport['failureClassifications'] = [];
833
- const repairAttempts: QaReport['repairAttempts'] = [];
834
- let intentInvariantPreserved = true;
835
-
836
- if (input.verify !== false && canRunPlaywright && pwTool && staticViolations.length === 0) {
837
- const maxIter = Math.min(Math.max(input.maxRepairIterations ?? 3, 1), 5);
838
- const originalSpecContents = new Map(specFiles.map((s) => [s.relativePath, s.content]));
839
- const started = Date.now();
840
- executed = true;
841
-
842
- for (let iter = 0; iter <= maxIter; iter += 1) {
843
- const runRes = await runCommandInWorkspace(
844
- pwTool.cmd,
845
- [...pwTool.prefixArgs, 'test', ...specFiles.map((s) => s.relativePath), '--trace=on', '--reporter=line'],
846
- workspaceRoot,
847
- 90_000,
848
- );
849
- execExitCode = runRes.exitCode;
850
- execStdout = (runRes.stdout + '\n' + runRes.stderr).slice(-2000);
851
- execDurationMs = Date.now() - started;
852
-
853
- if (execExitCode === 0) {
854
- execPassed = true;
855
- break;
856
- }
857
-
858
- // Diagnose failure using PIE's Trace/Diagnostic engine when trace.zip exists
859
- let category = 'UNKNOWN_ERROR';
860
- let summary = execStdout.slice(0, 300);
861
- const traceZip = await findFirstTraceZip(workspaceRoot);
862
- if (traceZip) {
863
- try {
864
- const { traceEntryName, networkEntryName, traceEntryNames, networkEntryNames } = ArchiveReader.findTraceFileEntries(traceZip);
865
- if (traceEntryName) {
866
- const traceText = await ArchiveReader.readEntriesAsText(traceZip, traceEntryNames ?? [traceEntryName]);
867
- const networkText = (networkEntryNames ?? (networkEntryName ? [networkEntryName] : [])).length > 0
868
- ? await ArchiveReader.readEntriesAsText(traceZip, networkEntryNames ?? [networkEntryName!])
869
- : '';
870
- const context = TraceParser.assembleDiagnosticContext(traceText.split('\n'), networkText.split('\n'));
871
- const correlated = EvidenceCorrelator.correlate(context);
872
- const has500 =
873
- context.networkEvents.some((n) => n.status >= 500) ||
874
- correlated.classification.category === 'BACKEND_500' ||
875
- explorationBundle.network.some((n) => n.status >= 500);
876
- category = has500 ? 'BACKEND_500' : correlated.classification.category;
877
- summary = context.failedAction.errorMessage ?? correlated.classification.reason ?? summary;
878
- }
879
- } catch {
880
- // Keep stdout fallback summary
881
- }
882
- }
883
- if (category === 'UNKNOWN_ERROR' && explorationBundle.network.some((n) => n.status >= 500)) {
884
- category = 'BACKEND_500';
885
- }
886
-
887
- const defectOwner: 'automation' | 'application' | 'environment' | 'unknown' =
888
- category === 'BACKEND_500' ? 'application' : category === 'LOCATOR_DRIFT' ? 'automation' : 'unknown';
889
- failureClassifications.push({ category, summary, defectOwner });
890
-
891
- // Never auto-repair application/backend defects or exceed repair budget
892
- if (defectOwner === 'application' || iter === maxIter || !input.repairCallback) {
893
- break;
894
- }
895
-
896
- const proposedPatch = await input.repairCallback({
897
- specFiles: candidate.files,
898
- failureCategory: category,
899
- errorText: summary,
900
- iteration: iter + 1,
901
- });
902
- if (!proposedPatch) break;
903
-
904
- // Enforce Assertion Preservation Invariant before accepting repaired spec!
905
- let invariantOk = true;
906
- for (const updated of proposedPatch) {
907
- const orig = originalSpecContents.get(updated.relativePath);
908
- if (orig && !assertionsPreserved(orig, updated.content)) {
909
- invariantOk = false;
910
- }
911
- }
912
-
913
- repairAttempts.push({
914
- iteration: iter + 1,
915
- category,
916
- action: invariantOk ? 'apply-repaired-files' : 'rejected-assertion-weakening',
917
- assertionsPreserved: invariantOk,
918
- succeeded: false,
919
- });
920
-
921
- if (!invariantOk) {
922
- intentInvariantPreserved = false;
923
- break;
924
- }
925
-
926
- await CandidateWriter.apply(workspaceRoot, { files: proposedPatch.map((f) => ({ ...f, action: 'MODIFY' })) }, { allowModify: true });
927
- candidate = { ...candidate, files: proposedPatch };
928
- const recheck = await runCommandInWorkspace(pwTool.cmd, [...pwTool.prefixArgs, 'test', ...specFiles.map((s) => s.relativePath), '--reporter=line'], workspaceRoot, 90_000);
929
- if (recheck.exitCode === 0) {
930
- repairAttempts[repairAttempts.length - 1]!.succeeded = true;
931
- execExitCode = 0;
932
- execPassed = true;
933
- break;
934
- }
935
- }
936
- }
937
-
938
- // Stage 3 — Mutation Falsification (Level C Assertions + Level B API Fault Injection)
939
- const mutationEntries: MutationResultEntry[] = [];
940
- let mutationExecuted = false;
941
-
942
- if (input.runMutations !== false && execPassed && canRunPlaywright && pwTool) {
943
- mutationExecuted = true;
944
- for (const spec of specFiles) {
945
- const targetPath = path.resolve(workspaceRoot, spec.relativePath);
946
- const original = await fs.readFile(targetPath, 'utf8');
947
- const levelCMutants = buildAssertionMutants(spec.relativePath, original);
948
-
949
- for (const mut of levelCMutants) {
950
- try {
951
- await fs.writeFile(targetPath, mut.mutatedContent, 'utf8');
952
- const res = await runCommandInWorkspace(pwTool.cmd, [...pwTool.prefixArgs, 'test', spec.relativePath, '--reporter=line'], workspaceRoot, 60_000);
953
- mutationEntries.push({
954
- id: mut.id,
955
- level: 'LEVEL_C_ASSERTION',
956
- specFile: spec.relativePath,
957
- description: mut.description,
958
- status: res.exitCode !== 0 ? 'KILLED' : 'SURVIVED',
959
- exitCode: res.exitCode,
960
- });
961
- } catch {
962
- mutationEntries.push({
963
- id: mut.id,
964
- level: 'LEVEL_C_ASSERTION',
965
- specFile: spec.relativePath,
966
- description: mut.description,
967
- status: 'INVALID',
968
- });
969
- } finally {
970
- await fs.writeFile(targetPath, original, 'utf8');
971
- }
972
- }
973
-
974
- // Level B: page.route() API fault injection if API calls were observed
975
- const hasApiEndpoints = explorationBundle.network.some((n) => n.url.includes('/api/'));
976
- if (hasApiEndpoints) {
977
- const injected = `test.beforeEach(async ({ page }) => {\n await page.route('**/api/**', (route) => route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: 'PIE_INJECTED_500' }) }));\n});\n`;
978
- const mutatedApiContent = original.includes('test.describe')
979
- ? original.replace(/(test\.describe\s*\()/, `${injected}\n$1`)
980
- : original.replace(/(test\s*\()/, `${injected}\n$1`);
981
- if (mutatedApiContent !== original) {
982
- try {
983
- await fs.writeFile(targetPath, mutatedApiContent, 'utf8');
984
- const res = await runCommandInWorkspace(pwTool.cmd, [...pwTool.prefixArgs, 'test', spec.relativePath, '--reporter=line'], workspaceRoot, 60_000);
985
- mutationEntries.push({
986
- id: 'mut-b-api-500',
987
- level: 'LEVEL_B_API_FAULT',
988
- specFile: spec.relativePath,
989
- description: `Inject HTTP 500 via page.route('**/api/**') in ${spec.relativePath}`,
990
- status: res.exitCode !== 0 ? 'KILLED' : 'SURVIVED',
991
- exitCode: res.exitCode,
992
- });
993
- } finally {
994
- await fs.writeFile(targetPath, original, 'utf8');
995
- }
996
- }
997
- }
998
- }
999
- }
1000
-
1001
- const killed = mutationEntries.filter((m) => m.status === 'KILLED').length;
1002
- const survived = mutationEntries.filter((m) => m.status === 'SURVIVED').length;
1003
- const invalid = mutationEntries.filter((m) => m.status === 'INVALID').length;
1004
- const equivalent = mutationEntries.filter((m) => m.status === 'EQUIVALENT').length;
1005
- const validTotal = killed + survived;
1006
- const mutationScore = validTotal > 0 ? Number(((killed / validTotal) * 100).toFixed(2)) : null;
1007
-
1008
- const hasDefects = explorationBundle.findings.some((f) => f.category === 'SPEC_VIOLATION' || f.category === 'RUNTIME_ERROR');
1009
- const finalStatus: QaReport['status'] = staticViolations.length > 0 || (executed && !execPassed)
1010
- ? 'VERIFICATION_FAILED'
1011
- : execPassed && (!hasDefects && (mutationScore === null || mutationScore >= 80))
1012
- ? 'VERIFIED'
1013
- : execPassed
1014
- ? 'VERIFIED_WITH_WARNINGS'
1015
- : 'GENERATED';
1016
-
1017
- const qaReportJsonPath = path.join(artifactDir, 'qa-report.json');
1018
- const qaReportMarkdownPath = path.join(artifactDir, 'qa-report.md');
1019
-
1020
- const qaReport: QaReport = {
1021
- reportVersion: '1.0',
1022
- generatedAt: new Date().toISOString(),
1023
- status: finalStatus,
1024
- featureName: featureTitle,
1025
- topology: explorationBundle.topology,
1026
- runtimeVerified: explorationBundle.runtimeVerified,
1027
- requirementCoverage: workflowPlan.requirements.map((r) => {
1028
- const relatedSc = workflowPlan.scenarios.filter((s) => s.requirementRefs.includes(r.id));
1029
- const statuses = relatedSc.map((s) => explorationBundle.scenarioStatuses.find((st) => st.scenarioId === s.id)?.status);
1030
- const covStatus = statuses.some((st) => st === 'BLOCKED' || st === 'DEFECT_DETECTED')
1031
- ? 'BLOCKED'
1032
- : statuses.some((st) => st === 'VERIFIED')
1033
- ? 'COVERED'
1034
- : 'NOT_COVERED';
1035
- return {
1036
- requirementId: r.id,
1037
- title: r.title,
1038
- status: covStatus,
1039
- scenarioIds: relatedSc.map((s) => s.id),
1040
- };
1041
- }),
1042
- scenarioCoverage: workflowPlan.scenarios.map((s) => ({
1043
- scenarioId: s.id,
1044
- title: s.title,
1045
- category: s.category,
1046
- applicability: s.applicability,
1047
- explorationStatus: explorationBundle.scenarioStatuses.find((st) => st.scenarioId === s.id)?.status,
1048
- })),
1049
- workflowStatuses: explorationBundle.scenarioStatuses,
1050
- generatedFiles: writtenFiles,
1051
- modifiedFiles,
1052
- staticValidation: {
1053
- passed: staticViolations.length === 0,
1054
- tscChecked,
1055
- ...(tscPassed !== undefined ? { tscPassed } : {}),
1056
- discoveryChecked,
1057
- ...(discoveryPassed !== undefined ? { discoveryPassed } : {}),
1058
- violations: staticViolations,
1059
- },
1060
- execution: {
1061
- executed,
1062
- passed: execPassed,
1063
- ...(execExitCode !== undefined ? { exitCode: execExitCode } : {}),
1064
- ...(execDurationMs !== undefined ? { durationMs: execDurationMs } : {}),
1065
- ...(execStdout ? { stdoutSnippet: execStdout } : {}),
1066
- },
1067
- failureClassifications,
1068
- repairAttempts,
1069
- intentInvariantPreserved,
1070
- mutationResults: {
1071
- executed: mutationExecuted,
1072
- total: mutationEntries.length,
1073
- validTotal,
1074
- killed,
1075
- survived,
1076
- invalid,
1077
- equivalent,
1078
- mutationScore,
1079
- entries: mutationEntries,
1080
- },
1081
- verificationLayers: {
1082
- UI: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'UI'),
1083
- API: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'API'),
1084
- CLIENT_RELOAD: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'CLIENT_RELOAD'),
1085
- DB: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'DB'),
1086
- },
1087
- findings: explorationBundle.findings,
1088
- unverifiedBehaviors: explorationBundle.scenarioStatuses.filter((s) => s.status !== 'VERIFIED').map((s) => s.scenarioId),
1089
- limitations: explorationBundle.limitations,
1090
- artifacts: {
1091
- workflowPlanPath,
1092
- featureFilePath,
1093
- explorationBundlePath,
1094
- qaReportJsonPath,
1095
- qaReportMarkdownPath,
1096
- },
1097
- environment: {
1098
- nodeVersion: process.version,
1099
- platform: process.platform,
1100
- workspaceRoot,
1101
- mcpDependenciesUsed: 'none',
1102
- },
1103
- };
1104
-
1105
- await fs.writeFile(qaReportJsonPath, `${JSON.stringify(qaReport, null, 2)}\n`, 'utf8');
1106
- await fs.writeFile(qaReportMarkdownPath, formatQaReportMarkdown(qaReport), 'utf8');
1107
-
1108
- return {
1109
- workflowPlan,
1110
- featureText,
1111
- explorationBundle,
1112
- candidate,
1113
- qaReport,
1114
- };
1115
- }
1
+ import fs from 'node:fs/promises';
2
+ import fsSync from 'node:fs';
3
+ import path from 'node:path';
4
+ import { z } from 'zod';
5
+ import { ArchiveReader } from '../trace/archive-reader.js';
6
+ import { TraceParser } from '../trace/trace-parser.js';
7
+ import { EvidenceCorrelator } from '../diagnostics/evidence-correlator.js';
8
+ import {
9
+ AuthorKind,
10
+ CandidateChange,
11
+ CandidateFile,
12
+ FullCodeGenerationProvider,
13
+ HybridAuthor,
14
+ LlmAuthor,
15
+ NamingLinter,
16
+ } from '../authoring/index.js';
17
+ import { AssertionQualityGate } from '../generator/assertion-quality-gate.js';
18
+ import { toClassName, toSlug } from '../generator/architecture-generator.js';
19
+ import { CandidateWriter } from '../generator/candidate-writer.js';
20
+ import { ExecutionRunner } from '../sandbox/execution-runner.js';
21
+ import {
22
+ ExplorationContextBundle,
23
+ ExplorationFinding,
24
+ ScenarioExplorationStatus,
25
+ saveExplorationContextBundle,
26
+ VerificationLayerStatus,
27
+ } from './evidence-bundle.js';
28
+ import { buildBrownfieldRepositoryGraph, BrownfieldRepositoryGraph } from './repository-profile.js';
29
+ import {
30
+ buildWorkflowPlan,
31
+ exportWorkflowToGherkin,
32
+ GenerationWorkflowPlan,
33
+ importGherkinToWorkflowPlan,
34
+ isStaleWorkflowPlan,
35
+ WorkflowPlanInput,
36
+ } from './workflow-planner.js';
37
+ import {
38
+ ControlledLoginConfig,
39
+ DbVerificationResult,
40
+ InteractiveWorkflowRunner,
41
+ } from '../reproduce/interactive-workflow-runner.js';
42
+
43
+ export interface MutationResultEntry {
44
+ id: string;
45
+ level: 'LEVEL_C_ASSERTION' | 'LEVEL_B_API_FAULT';
46
+ specFile: string;
47
+ description: string;
48
+ status: 'KILLED' | 'SURVIVED' | 'INVALID' | 'EQUIVALENT';
49
+ exitCode?: number | undefined;
50
+ }
51
+
52
+ export interface QaReport {
53
+ reportVersion: '1.0';
54
+ generatedAt: string;
55
+ status: 'VERIFIED' | 'VERIFIED_WITH_WARNINGS' | 'GENERATED' | 'NEEDS_INPUT' | 'ABSTAINED' | 'VERIFICATION_FAILED';
56
+ featureName: string;
57
+ topology: ExplorationContextBundle['topology'];
58
+ runtimeVerified: boolean;
59
+ requirementCoverage: Array<{
60
+ requirementId: string;
61
+ title: string;
62
+ status: 'COVERED' | 'BLOCKED' | 'NOT_APPLICABLE' | 'NOT_COVERED';
63
+ scenarioIds: string[];
64
+ }>;
65
+ scenarioCoverage: Array<{
66
+ scenarioId: string;
67
+ title: string;
68
+ category: string;
69
+ applicability: string;
70
+ explorationStatus?: string | undefined;
71
+ }>;
72
+ workflowStatuses: ScenarioExplorationStatus[];
73
+ generatedFiles: string[];
74
+ modifiedFiles: string[];
75
+ staticValidation: {
76
+ passed: boolean;
77
+ tscChecked: boolean;
78
+ tscPassed?: boolean | undefined;
79
+ discoveryChecked: boolean;
80
+ discoveryPassed?: boolean | undefined;
81
+ violations: string[];
82
+ };
83
+ execution: {
84
+ executed: boolean;
85
+ passed: boolean;
86
+ exitCode?: number | undefined;
87
+ durationMs?: number | undefined;
88
+ stdoutSnippet?: string | undefined;
89
+ };
90
+ failureClassifications: Array<{
91
+ category: string;
92
+ summary: string;
93
+ defectOwner: 'automation' | 'application' | 'environment' | 'unknown';
94
+ }>;
95
+ repairAttempts: Array<{
96
+ iteration: number;
97
+ category: string;
98
+ action: string;
99
+ assertionsPreserved: boolean;
100
+ succeeded: boolean;
101
+ }>;
102
+ intentInvariantPreserved: boolean;
103
+ mutationResults: {
104
+ executed: boolean;
105
+ total: number;
106
+ validTotal: number;
107
+ killed: number;
108
+ survived: number;
109
+ invalid: number;
110
+ equivalent: number;
111
+ mutationScore: number | null;
112
+ entries: MutationResultEntry[];
113
+ };
114
+ verificationLayers: {
115
+ UI: VerificationLayerStatus;
116
+ API: VerificationLayerStatus;
117
+ CLIENT_RELOAD: VerificationLayerStatus;
118
+ DB: VerificationLayerStatus;
119
+ };
120
+ findings: ExplorationFinding[];
121
+ unverifiedBehaviors: string[];
122
+ limitations: string[];
123
+ artifacts: {
124
+ workflowPlanPath?: string | undefined;
125
+ featureFilePath?: string | undefined;
126
+ explorationBundlePath?: string | undefined;
127
+ qaReportJsonPath?: string | undefined;
128
+ qaReportMarkdownPath?: string | undefined;
129
+ };
130
+ environment: {
131
+ nodeVersion: string;
132
+ platform: string;
133
+ workspaceRoot: string;
134
+ mcpDependenciesUsed: 'none';
135
+ };
136
+ }
137
+
138
+ export interface EvidenceDrivenGenerationInput extends WorkflowPlanInput {
139
+ workspaceRoot: string;
140
+ outArtifactDir?: string | undefined;
141
+ author?: AuthorKind | undefined;
142
+ fullCodeProvider?: FullCodeGenerationProvider | undefined;
143
+ editedFeatureText?: string | undefined;
144
+ storageStatePath?: string | undefined;
145
+ loginConfig?: ControlledLoginConfig | undefined;
146
+ allowMutatingActions?: boolean | undefined;
147
+ allowDestructiveActions?: boolean | undefined;
148
+ designatedTestEnvironment?: boolean | undefined;
149
+ dbVerifier?: ((scenario: any, actions: any[]) => Promise<DbVerificationResult>) | undefined;
150
+ write?: boolean | undefined;
151
+ verify?: boolean | undefined;
152
+ runMutations?: boolean | undefined;
153
+ maxRepairIterations?: number | undefined;
154
+ repairCallback?: ((context: {
155
+ specFiles: CandidateFile[];
156
+ failureCategory: string;
157
+ errorText: string;
158
+ iteration: number;
159
+ }) => Promise<CandidateFile[] | undefined>) | undefined;
160
+ }
161
+
162
+ export interface EvidenceDrivenGenerationResult {
163
+ workflowPlan: GenerationWorkflowPlan;
164
+ featureText: string;
165
+ explorationBundle: ExplorationContextBundle;
166
+ candidate: CandidateChange;
167
+ qaReport: QaReport;
168
+ }
169
+
170
+ export const QaReportSchema = z.object({
171
+ reportVersion: z.literal('1.0'),
172
+ generatedAt: z.string(),
173
+ status: z.enum(['VERIFIED', 'VERIFIED_WITH_WARNINGS', 'GENERATED', 'NEEDS_INPUT', 'ABSTAINED', 'VERIFICATION_FAILED']),
174
+ featureName: z.string().min(1),
175
+ topology: z.enum(['monorepo', 'standalone', 'source-only']),
176
+ runtimeVerified: z.boolean(),
177
+ });
178
+
179
+ function extractExpectLines(content: string): string[] {
180
+ return content
181
+ .split(/\r?\n/)
182
+ .map((line) => line.trim())
183
+ .filter((line) => /\bexpect(?:\.soft|\.poll)?\s*\(/.test(line));
184
+ }
185
+
186
+ export function assertionsPreserved(originalSpec: string, repairedSpec: string): boolean {
187
+ const orig = extractExpectLines(originalSpec);
188
+ if (orig.length === 0) return true;
189
+ const rep = extractExpectLines(repairedSpec);
190
+ if (rep.length < orig.length) return false;
191
+
192
+ const repJoined = rep.join('\n');
193
+ for (const line of orig) {
194
+ // Reject replacing specific locator assertions with generic body/html checks
195
+ if (!/locator\s*\(\s*['"]body['"]\s*\)/i.test(line) && /expect\s*\(\s*page\.locator\s*\(\s*['"]body['"]\s*\)\s*\)/i.test(repJoined)) {
196
+ return false;
197
+ }
198
+
199
+ // Preserve every semantic expected target literal inside getByText/getByRole/getByTestId/getByLabel
200
+ const semanticLiterals = [...line.matchAll(/(?:getByText|getByTestId|getByLabel|getByPlaceholder)\s*\(\s*(['"`][^'"`]+['"`])/g)];
201
+ for (const m of semanticLiterals) {
202
+ if (m[1] && !repJoined.includes(m[1])) {
203
+ return false;
204
+ }
205
+ }
206
+ const roleNames = [...line.matchAll(/name\s*:\s*(['"`][^'"`]+['"`]|\/[^/]+\/[gimuy]*)/g)];
207
+ for (const m of roleNames) {
208
+ if (m[1] && !repJoined.includes(m[1])) {
209
+ return false;
210
+ }
211
+ }
212
+
213
+ // Preserve every expected outcome matcher and expected argument value
214
+ const matcherMatch = line.match(/\)\.(to[A-Za-z]+)\(([^)]*)\)/);
215
+ if (matcherMatch) {
216
+ const matcherName = matcherMatch[1]!;
217
+ const expectedArg = matcherMatch[2]!.trim();
218
+ if (expectedArg && !repJoined.includes(expectedArg)) {
219
+ return false;
220
+ }
221
+ if (!repJoined.includes(`.${matcherName}(`)) {
222
+ return false;
223
+ }
224
+ }
225
+ }
226
+ return true;
227
+ }
228
+
229
+ function summarizeLayerStatus(statuses: ScenarioExplorationStatus[], layer: keyof ScenarioExplorationStatus['verificationLayers']): VerificationLayerStatus {
230
+ const vals = statuses.map((s) => s.verificationLayers[layer]);
231
+ if (vals.includes('FAIL')) return 'FAIL';
232
+ if (vals.includes('PASS')) return 'PASS';
233
+ if (vals.includes('NOT_VERIFIED')) return 'NOT_VERIFIED';
234
+ return 'NOT_APPLICABLE';
235
+ }
236
+
237
+ function synthesizeGroundedCandidateFromBundle(
238
+ featureName: string,
239
+ workflowPlan: GenerationWorkflowPlan,
240
+ bundle: ExplorationContextBundle,
241
+ repoGraph?: BrownfieldRepositoryGraph,
242
+ ): CandidateChange {
243
+ const slug = toSlug(featureName);
244
+ const rawBase = toClassName(featureName);
245
+ const pageClassName = rawBase.endsWith('Page') ? rawBase : `${rawBase}Page`;
246
+ const pageVar = pageClassName[0]!.toLowerCase() + pageClassName.slice(1);
247
+
248
+ const validScenarios = workflowPlan.scenarios.filter((sc) => {
249
+ const st = bundle.scenarioStatuses.find((s) => s.scenarioId === sc.id);
250
+ return (
251
+ sc.applicability === 'applicable' &&
252
+ st?.status !== 'BLOCKED' &&
253
+ st?.status !== 'DEFECT_DETECTED'
254
+ );
255
+ });
256
+
257
+ const existingFilesSet = new Set<string>([
258
+ ...(repoGraph?.configPaths ?? []),
259
+ ...(repoGraph?.pomExports.map((p) => p.sourceFile) ?? []),
260
+ ]);
261
+
262
+ const navUrl = bundle.baseUrl ? new URL(bundle.baseUrl).pathname || '/' : '/';
263
+ const routeLocators = bundle.locators.filter((l) => !l.route || l.route === navUrl);
264
+ const scopedLocators = routeLocators.length > 0 ? routeLocators : bundle.locators;
265
+
266
+ const inputLoc = scopedLocators.find((l) => (l.strategy === 'placeholder' || l.role === 'textbox') && l.visible);
267
+ const submitLoc = scopedLocators.find((l) => l.role === 'button' && !/delete|remove|cancel|close/i.test(l.accessibleName ?? '') && l.visible);
268
+ const toggleLoc = scopedLocators.find((l) => l.role === 'checkbox' && l.visible);
269
+
270
+ const pomLocators: string[] = [];
271
+ const pomAssigns: string[] = [];
272
+ const pomMethods: string[] = [];
273
+ const locatorExpr = (expr: string) => expr.replace(/^page\./, 'this.page.');
274
+
275
+ if (inputLoc) {
276
+ pomLocators.push(' readonly primaryInput: Locator;');
277
+ pomAssigns.push(` this.primaryInput = ${locatorExpr(inputLoc.expression)};`);
278
+ }
279
+ if (submitLoc) {
280
+ pomLocators.push(' readonly submitButton: Locator;');
281
+ pomAssigns.push(` this.submitButton = ${locatorExpr(submitLoc.expression)};`);
282
+ }
283
+ if (toggleLoc) {
284
+ pomLocators.push(' readonly itemToggle: Locator;');
285
+ pomAssigns.push(` this.itemToggle = ${locatorExpr(toggleLoc.expression)};`);
286
+ }
287
+
288
+ if (inputLoc && submitLoc) {
289
+ pomMethods.push(` async submitItem(value: string): Promise<void> {
290
+ await this.primaryInput.fill(value);
291
+ await this.submitButton.click();
292
+ }`);
293
+ } else if (inputLoc) {
294
+ pomMethods.push(` async submitItem(value: string): Promise<void> {
295
+ await this.primaryInput.fill(value);
296
+ await this.primaryInput.press('Enter');
297
+ }`);
298
+ } else if (submitLoc) {
299
+ pomMethods.push(` async clickPrimaryAction(): Promise<void> {
300
+ await this.submitButton.click();
301
+ }`);
302
+ }
303
+
304
+ if (submitLoc) {
305
+ pomMethods.push(` async submitEmpty(): Promise<void> {
306
+ await this.submitButton.click();
307
+ }`);
308
+ }
309
+
310
+ const pomContent = `import type { Page, Locator } from '@playwright/test';
311
+
312
+ export class ${pageClassName} {
313
+ ${pomLocators.length > 0 ? `${pomLocators.join('\n')}\n` : ''}
314
+ constructor(public readonly page: Page) {
315
+ ${pomAssigns.join('\n')}
316
+ }
317
+
318
+ async goto(): Promise<void> {
319
+ await this.page.goto(${JSON.stringify(navUrl)});
320
+ }
321
+ ${pomMethods.length > 0 ? `\n${pomMethods.join('\n\n')}\n` : ''}}
322
+ `;
323
+
324
+ const hasExistingFixture = Boolean(repoGraph?.testImport && repoGraph.testImport !== '@playwright/test');
325
+ const fixturePath = 'tests/fixtures/app.fixtures.ts';
326
+ const specPath = `tests/${slug}.spec.ts`;
327
+ const pomPath = `tests/pages/${pageClassName}.ts`;
328
+
329
+ let fixtureImportForSpec = '@playwright/test';
330
+ const candidateFiles: CandidateFile[] = [
331
+ {
332
+ relativePath: pomPath,
333
+ action: 'CREATE',
334
+ content: pomContent,
335
+ purpose: 'Evidence-grounded Page Object Model reusing observed locators.',
336
+ evidenceReferences: [inputLoc?.id, submitLoc?.id].filter((x): x is string => Boolean(x)),
337
+ },
338
+ ];
339
+
340
+ const fixtureContent = `import { test as base } from '@playwright/test';
341
+ import { ${pageClassName} } from '../pages/${pageClassName}.js';
342
+
343
+ type ${pageClassName}Fixtures = {
344
+ ${pageVar}: ${pageClassName};
345
+ };
346
+
347
+ export const test = base.extend<${pageClassName}Fixtures>({
348
+ ${pageVar}: async ({ page }, use) => {
349
+ const ${pageVar} = new ${pageClassName}(page);
350
+ await ${pageVar}.goto();
351
+ await use(${pageVar});
352
+ },
353
+ });
354
+
355
+ export { expect } from '@playwright/test';
356
+ `;
357
+
358
+ if (!hasExistingFixture) {
359
+ candidateFiles.push({
360
+ relativePath: fixturePath,
361
+ action: 'CREATE',
362
+ content: fixtureContent,
363
+ purpose: 'Composable fixture injecting feature Page Object.',
364
+ });
365
+ fixtureImportForSpec = `./fixtures/app.fixtures.js`;
366
+ } else {
367
+ const relFromSpec = path.posix.relative('tests', repoGraph!.testImport).replace(/\.ts$/, '.js');
368
+ fixtureImportForSpec = relFromSpec.startsWith('.') ? relFromSpec : `./${relFromSpec}`;
369
+ }
370
+
371
+ const testBlocks: string[] = [];
372
+
373
+ for (const sc of validScenarios) {
374
+ const varTitle = `Test-${sc.id.slice(0, 10)}`;
375
+ if (sc.category === 'validation' || sc.category === 'negative' || sc.category === 'required-fields') {
376
+ if (inputLoc && submitLoc) {
377
+ testBlocks.push(hasExistingFixture
378
+ ? ` test(${JSON.stringify(sc.title)}, async ({ page }) => {
379
+ const pageObj = new ${pageClassName}(page);
380
+ await pageObj.goto();
381
+ const marker = ${JSON.stringify(`__SHOULD_NOT_EXIST_${sc.id}__`)};
382
+ await pageObj.primaryInput.fill('');
383
+ await pageObj.submitEmpty();
384
+ await expect(page.getByText(marker)).toBeHidden();
385
+ });`
386
+ : ` test(${JSON.stringify(sc.title)}, async ({ ${pageVar}, page }) => {
387
+ const marker = ${JSON.stringify(`__SHOULD_NOT_EXIST_${sc.id}__`)};
388
+ await ${pageVar}.primaryInput.fill('');
389
+ await ${pageVar}.submitEmpty();
390
+ await expect(page.getByText(marker)).toBeHidden();
391
+ });`);
392
+ }
393
+ } else if (sc.category === 'persistence') {
394
+ if (inputLoc) {
395
+ testBlocks.push(hasExistingFixture
396
+ ? ` test(${JSON.stringify(sc.title)}, async ({ page }) => {
397
+ const pageObj = new ${pageClassName}(page);
398
+ await pageObj.goto();
399
+ const title = ${JSON.stringify(`${varTitle}-persist`)};
400
+ await pageObj.submitItem(title);
401
+ await expect(page.getByText(title)).toBeVisible();
402
+ await page.reload();
403
+ await expect(page.getByText(title)).toBeVisible();
404
+ });`
405
+ : ` test(${JSON.stringify(sc.title)}, async ({ ${pageVar}, page }) => {
406
+ const title = ${JSON.stringify(`${varTitle}-persist`)};
407
+ await ${pageVar}.submitItem(title);
408
+ await expect(page.getByText(title)).toBeVisible();
409
+ await page.reload();
410
+ await expect(page.getByText(title)).toBeVisible();
411
+ });`);
412
+ }
413
+ } else if (inputLoc) {
414
+ testBlocks.push(hasExistingFixture
415
+ ? ` test(${JSON.stringify(sc.title)}, async ({ page }) => {
416
+ const pageObj = new ${pageClassName}(page);
417
+ await pageObj.goto();
418
+ const title = ${JSON.stringify(`${varTitle}-item`)};
419
+ await pageObj.submitItem(title);
420
+ await expect(page.getByText(title)).toBeVisible();
421
+ });`
422
+ : ` test(${JSON.stringify(sc.title)}, async ({ ${pageVar}, page }) => {
423
+ const title = ${JSON.stringify(`${varTitle}-item`)};
424
+ await ${pageVar}.submitItem(title);
425
+ await expect(page.getByText(title)).toBeVisible();
426
+ });`);
427
+ } else if (submitLoc) {
428
+ testBlocks.push(hasExistingFixture
429
+ ? ` test(${JSON.stringify(sc.title)}, async ({ page }) => {
430
+ const pageObj = new ${pageClassName}(page);
431
+ await pageObj.goto();
432
+ await pageObj.clickPrimaryAction();
433
+ await expect(page.locator('#nav-cart, main').first()).toBeVisible();
434
+ });`
435
+ : ` test(${JSON.stringify(sc.title)}, async ({ ${pageVar}, page }) => {
436
+ await ${pageVar}.clickPrimaryAction();
437
+ await expect(page.locator('main').first()).toBeVisible();
438
+ });`);
439
+ }
440
+ }
441
+
442
+ // Ensure non-empty behavioral spec only when we had verified controls
443
+ const specContent = `import { test, expect } from ${JSON.stringify(fixtureImportForSpec)};
444
+ ${hasExistingFixture ? `import { ${pageClassName} } from ${JSON.stringify(`./pages/${pageClassName}.js`)};\n` : ''}
445
+ test.describe(${JSON.stringify(`${featureName} verified workflows`)}, () => {
446
+ ${testBlocks.join('\n\n')}
447
+ });
448
+ `;
449
+
450
+ candidateFiles.push({
451
+ relativePath: specPath,
452
+ action: 'CREATE',
453
+ content: specContent,
454
+ purpose: 'Evidence-grounded Playwright test specification.',
455
+ scenarioIds: validScenarios.map((s) => s.id),
456
+ });
457
+
458
+ if (!existingFilesSet.has('playwright.config.ts') && (repoGraph?.configPaths.length ?? 0) === 0) {
459
+ candidateFiles.push({
460
+ relativePath: 'playwright.config.ts',
461
+ action: 'CREATE',
462
+ content: `import { defineConfig, devices } from '@playwright/test';
463
+ export default defineConfig({
464
+ testDir: './tests',
465
+ fullyParallel: false,
466
+ reporter: [['list']],
467
+ use: {
468
+ baseURL: ${JSON.stringify(bundle.baseUrl ?? 'http://localhost:3000')},
469
+ trace: 'on',
470
+ },
471
+ projects: [{ name: 'chromium', use: { ...devices['Desktop Chrome'] } }],
472
+ });
473
+ `,
474
+ purpose: 'Greenfield Playwright configuration.',
475
+ });
476
+ }
477
+
478
+ const canBuild = testBlocks.length > 0;
479
+ return {
480
+ files: canBuild ? candidateFiles : [],
481
+ symbols: {
482
+ planVersion: '1',
483
+ pageClassName,
484
+ pageVariable: pageVar,
485
+ symbols: [],
486
+ assumptions: [],
487
+ },
488
+ rationale: canBuild
489
+ ? 'Evidence-grounded deterministic fallback suite synthesized from ExplorationContextBundle.'
490
+ : 'Insufficient verified interactive controls or all applicable scenarios blocked.',
491
+ authorKind: 'hybrid',
492
+ featureName,
493
+ slug,
494
+ mode: hasExistingFixture ? 'brownfield' : 'greenfield',
495
+ status: canBuild ? 'GENERATE' : 'ABSTAIN',
496
+ fullCodeAuthoring: true,
497
+ usedFallback: true,
498
+ };
499
+ }
500
+
501
+ function resolveWorkspaceCli(
502
+ workspaceRoot: string,
503
+ tool: 'tsc' | 'playwright',
504
+ ): { cmd: string; prefixArgs: string[] } | undefined {
505
+ if (tool === 'tsc') {
506
+ const jsEntry = path.join(workspaceRoot, 'node_modules', 'typescript', 'bin', 'tsc');
507
+ if (fsSync.existsSync(jsEntry)) {
508
+ return { cmd: process.execPath, prefixArgs: [jsEntry] };
509
+ }
510
+ const binEntry = path.join(workspaceRoot, 'node_modules', '.bin', process.platform === 'win32' ? 'tsc.cmd' : 'tsc');
511
+ if (fsSync.existsSync(binEntry)) {
512
+ return { cmd: 'npx', prefixArgs: ['tsc'] };
513
+ }
514
+ return undefined;
515
+ }
516
+
517
+ const pwJs = path.join(workspaceRoot, 'node_modules', '@playwright', 'test', 'cli.js');
518
+ if (fsSync.existsSync(pwJs)) {
519
+ return { cmd: process.execPath, prefixArgs: [pwJs] };
520
+ }
521
+ const pwBin = path.join(workspaceRoot, 'node_modules', '.bin', process.platform === 'win32' ? 'playwright.cmd' : 'playwright');
522
+ if (fsSync.existsSync(pwBin)) {
523
+ return { cmd: 'npx', prefixArgs: ['playwright'] };
524
+ }
525
+ return undefined;
526
+ }
527
+
528
+ async function runCommandInWorkspace(
529
+ cmd: string,
530
+ args: string[],
531
+ cwd: string,
532
+ timeoutMs = 60_000,
533
+ ): Promise<{ exitCode: number; stdout: string; stderr: string }> {
534
+ const res = await ExecutionRunner.run(cmd, {
535
+ cwd,
536
+ args,
537
+ timeoutMs,
538
+ });
539
+ return { exitCode: res.exitCode ?? 1, stdout: res.stdout, stderr: res.stderr };
540
+ }
541
+
542
+ async function findFirstTraceZip(root: string): Promise<string | undefined> {
543
+ const candidates = [path.join(root, 'test-results'), path.join(root, '.playwright-results')];
544
+ for (const base of candidates) {
545
+ if (!fsSync.existsSync(base)) continue;
546
+ const entries = await fs.readdir(base, { withFileTypes: true }).catch(() => []);
547
+ for (const entry of entries) {
548
+ if (entry.isDirectory()) {
549
+ const traceFile = path.join(base, entry.name, 'trace.zip');
550
+ if (fsSync.existsSync(traceFile)) return traceFile;
551
+ } else if (entry.name.endsWith('trace.zip')) {
552
+ return path.join(base, entry.name);
553
+ }
554
+ }
555
+ }
556
+ return undefined;
557
+ }
558
+
559
+ function buildAssertionMutants(specPath: string, content: string): Array<{ id: string; description: string; mutatedContent: string }> {
560
+ const mutants: Array<{ id: string; description: string; mutatedContent: string }> = [];
561
+
562
+ if (content.includes('.toBeVisible()')) {
563
+ mutants.push({
564
+ id: 'mut-c-invert-visible',
565
+ description: `Invert .toBeVisible() -> .toBeHidden() in ${specPath}`,
566
+ mutatedContent: content.replace('.toBeVisible()', '.toBeHidden()'),
567
+ });
568
+ }
569
+ if (content.includes('.toBeHidden()')) {
570
+ mutants.push({
571
+ id: 'mut-c-invert-hidden',
572
+ description: `Invert .toBeHidden() -> .toBeVisible() in ${specPath}`,
573
+ mutatedContent: content.replace('.toBeHidden()', '.toBeVisible()'),
574
+ });
575
+ }
576
+ const textMatch = content.match(/expect\([^)]*\.getByText\(([^)]+)\)\)/);
577
+ if (textMatch) {
578
+ mutants.push({
579
+ id: 'mut-c-expected-text',
580
+ description: `Corrupt getByText() expected target in ${specPath}`,
581
+ mutatedContent: content.replace(textMatch[0], `expect(page.getByText("__PIE_MUTATED_EXPECTATION__"))`),
582
+ });
583
+ }
584
+ const countMatch = content.match(/\.toHaveCount\((\d+)\)/);
585
+ if (countMatch) {
586
+ const mutatedCount = Number(countMatch[1]) + 99;
587
+ mutants.push({
588
+ id: 'mut-c-count',
589
+ description: `Mutate .toHaveCount(${countMatch[1]}) -> .toHaveCount(${mutatedCount}) in ${specPath}`,
590
+ mutatedContent: content.replace(countMatch[0], `.toHaveCount(${mutatedCount})`),
591
+ });
592
+ }
593
+ return mutants;
594
+ }
595
+
596
+ export function formatQaReportMarkdown(report: QaReport): string {
597
+ const lines: string[] = [
598
+ `# PIE QA Verification Report — ${report.featureName}`,
599
+ '',
600
+ `- **Status:** \`${report.status}\``,
601
+ `- **Topology:** \`${report.topology}\``,
602
+ `- **Runtime Verified:** \`${report.runtimeVerified}\``,
603
+ `- **Generated:** ${report.generatedAt}`,
604
+ `- **MCP Dependencies:** \`${report.environment.mcpDependenciesUsed}\``,
605
+ '',
606
+ '## Verification Layers',
607
+ '| Layer | Status |',
608
+ '|---|---|',
609
+ `| UI | ${report.verificationLayers.UI} |`,
610
+ `| API | ${report.verificationLayers.API} |`,
611
+ `| CLIENT_RELOAD | ${report.verificationLayers.CLIENT_RELOAD} |`,
612
+ `| DB | ${report.verificationLayers.DB} |`,
613
+ '',
614
+ '## Generated & Modified Files',
615
+ ...report.generatedFiles.map((f) => `- CREATE \`${f}\``),
616
+ ...report.modifiedFiles.map((f) => `- MODIFY \`${f}\``),
617
+ '',
618
+ '## Mutation Falsification',
619
+ report.mutationResults.executed
620
+ ? `- **Score:** ${report.mutationResults.mutationScore ?? 'N/A'}% (${report.mutationResults.killed}/${report.mutationResults.validTotal} valid mutants killed, ${report.mutationResults.survived} survived, ${report.mutationResults.invalid} invalid)`
621
+ : '- Mutation testing skipped.',
622
+ '',
623
+ '## Findings',
624
+ ...(report.findings.length > 0
625
+ ? report.findings.map((f) => `- **[${f.category}]** (${f.severity}) ${f.actualBehavior}`)
626
+ : ['- None']),
627
+ ];
628
+ return `${lines.join('\n')}\n`;
629
+ }
630
+
631
+ export async function runEvidenceDrivenGeneration(
632
+ input: EvidenceDrivenGenerationInput,
633
+ ): Promise<EvidenceDrivenGenerationResult> {
634
+ const workspaceRoot = path.resolve(input.workspaceRoot);
635
+ const artifactDir = path.resolve(input.outArtifactDir ?? path.join(workspaceRoot, '.pie'));
636
+ await fs.mkdir(artifactDir, { recursive: true });
637
+
638
+ // Phase 1 — Repo graph hints + Requirement intake + Workflow Plan + Gherkin
639
+ const repoGraph = await buildBrownfieldRepositoryGraph(workspaceRoot).catch(() => undefined);
640
+ let workflowPlan = buildWorkflowPlan({
641
+ ...input,
642
+ ...(repoGraph ? { sourceHints: repoGraph.sourceHints } : {}),
643
+ });
644
+ let featureText = exportWorkflowToGherkin(workflowPlan);
645
+
646
+ if (input.editedFeatureText) {
647
+ if (isStaleWorkflowPlan(input.editedFeatureText, workflowPlan)) {
648
+ const imported = importGherkinToWorkflowPlan(input.editedFeatureText, workflowPlan);
649
+ if (!imported.success || !imported.plan) {
650
+ throw new Error(`Invalid edited Gherkin feature: ${imported.errors.join('; ')}`);
651
+ }
652
+ workflowPlan = imported.plan;
653
+ featureText = input.editedFeatureText;
654
+ }
655
+ }
656
+
657
+ const workflowPlanPath = path.join(artifactDir, 'workflow-plan.json');
658
+ const featureFilePath = path.join(artifactDir, `${workflowPlan.slug}.feature`);
659
+ await fs.writeFile(workflowPlanPath, `${JSON.stringify(workflowPlan, null, 2)}\n`, 'utf8');
660
+ await fs.writeFile(featureFilePath, featureText, 'utf8');
661
+
662
+ // Phase 2 — Stateful Live Exploration & Verification
663
+ const explorationBundlePath = path.join(artifactDir, 'exploration-evidence.json');
664
+ const explorationBundle = await InteractiveWorkflowRunner.run({
665
+ baseUrl: input.baseUrl,
666
+ workspaceRoot,
667
+ workflowPlan,
668
+ checkpointPath: explorationBundlePath,
669
+ storageStatePath: input.storageStatePath,
670
+ loginConfig: input.loginConfig,
671
+ allowMutatingActions: input.allowMutatingActions ?? true,
672
+ allowDestructiveActions: input.allowDestructiveActions,
673
+ designatedTestEnvironment: input.designatedTestEnvironment,
674
+ dbVerifier: input.dbVerifier,
675
+ repositoryGraph: repoGraph,
676
+ });
677
+ await saveExplorationContextBundle(explorationBundlePath, explorationBundle);
678
+
679
+ const allowedEvidenceIds = [...new Set([
680
+ ...explorationBundle.evidenceIndex.map((e) => e.id),
681
+ ...explorationBundle.actions.map((a) => a.id),
682
+ ...explorationBundle.locators.map((l) => l.id),
683
+ ...explorationBundle.network.map((n) => n.id),
684
+ ...explorationBundle.findings.map((f) => f.id),
685
+ ...explorationBundle.scenarioStatuses.map((s) => s.scenarioId),
686
+ ])];
687
+
688
+ // Phase 3 — Authoring (Tier-L2 LlmAuthor if configured/supplied, else grounded synthesis / HybridAuthor)
689
+ let candidate: CandidateChange;
690
+ const featureTitle = workflowPlan.featureName;
691
+
692
+ if (input.author === 'llm' || input.fullCodeProvider) {
693
+ const llmAuthor = new LlmAuthor({ provider: input.fullCodeProvider });
694
+ const rawCandidate = await llmAuthor.propose({
695
+ featureName: featureTitle,
696
+ mode: repoGraph && repoGraph.configPaths.length > 0 ? 'brownfield' : 'greenfield',
697
+ baseUrl: input.baseUrl,
698
+ requirements: workflowPlan.requirements.map((r) => `${r.id}: ${r.description}`).join('\n'),
699
+ workflowPlan,
700
+ explorationBundle,
701
+ contextPacket: { repoGraph, explorationBundle },
702
+ allowedEvidenceIds,
703
+ fullCodeProvider: input.fullCodeProvider,
704
+ });
705
+ if (rawCandidate.usedFallback) {
706
+ candidate = synthesizeGroundedCandidateFromBundle(featureTitle, workflowPlan, explorationBundle, repoGraph);
707
+ candidate.usedFallback = true;
708
+ candidate.authorKind = rawCandidate.authorKind;
709
+ candidate.rationale = `${rawCandidate.rationale} (synthesized via exploration-grounded fallback)`;
710
+ } else {
711
+ candidate = rawCandidate;
712
+ }
713
+ } else if (explorationBundle.locators.length > 0) {
714
+ candidate = synthesizeGroundedCandidateFromBundle(featureTitle, workflowPlan, explorationBundle, repoGraph);
715
+ } else {
716
+ const hybrid = new HybridAuthor();
717
+ candidate = await hybrid.propose({
718
+ featureName: featureTitle,
719
+ mode: repoGraph && repoGraph.configPaths.length > 0 ? 'brownfield' : 'greenfield',
720
+ baseUrl: input.baseUrl,
721
+ requirements: workflowPlan.requirements.map((r) => r.description).join('\n'),
722
+ });
723
+ }
724
+
725
+ // Handle NEEDS_INPUT / ABSTAIN cleanly
726
+ if (candidate.status === 'NEEDS_INPUT' || candidate.status === 'ABSTAIN' || candidate.files.length === 0) {
727
+ const report: QaReport = {
728
+ reportVersion: '1.0',
729
+ generatedAt: new Date().toISOString(),
730
+ status: candidate.status === 'NEEDS_INPUT' ? 'NEEDS_INPUT' : 'ABSTAINED',
731
+ featureName: featureTitle,
732
+ topology: explorationBundle.topology,
733
+ runtimeVerified: explorationBundle.runtimeVerified,
734
+ requirementCoverage: workflowPlan.requirements.map((r) => ({
735
+ requirementId: r.id,
736
+ title: r.title,
737
+ status: 'NOT_COVERED',
738
+ scenarioIds: workflowPlan.scenarios.filter((s) => s.requirementRefs.includes(r.id)).map((s) => s.id),
739
+ })),
740
+ scenarioCoverage: workflowPlan.scenarios.map((s) => ({
741
+ scenarioId: s.id,
742
+ title: s.title,
743
+ category: s.category,
744
+ applicability: s.applicability,
745
+ explorationStatus: explorationBundle.scenarioStatuses.find((st) => st.scenarioId === s.id)?.status,
746
+ })),
747
+ workflowStatuses: explorationBundle.scenarioStatuses,
748
+ generatedFiles: [],
749
+ modifiedFiles: [],
750
+ staticValidation: { passed: false, tscChecked: false, discoveryChecked: false, violations: candidate.validationErrors ?? [candidate.rationale] },
751
+ execution: { executed: false, passed: false },
752
+ failureClassifications: [],
753
+ repairAttempts: [],
754
+ intentInvariantPreserved: true,
755
+ mutationResults: { executed: false, total: 0, validTotal: 0, killed: 0, survived: 0, invalid: 0, equivalent: 0, mutationScore: null, entries: [] },
756
+ verificationLayers: {
757
+ UI: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'UI'),
758
+ API: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'API'),
759
+ CLIENT_RELOAD: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'CLIENT_RELOAD'),
760
+ DB: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'DB'),
761
+ },
762
+ findings: explorationBundle.findings,
763
+ unverifiedBehaviors: workflowPlan.scenarios.map((s) => s.title),
764
+ limitations: [...explorationBundle.limitations, candidate.rationale],
765
+ artifacts: { workflowPlanPath, featureFilePath, explorationBundlePath },
766
+ environment: { nodeVersion: process.version, platform: process.platform, workspaceRoot, mcpDependenciesUsed: 'none' },
767
+ };
768
+ return { workflowPlan, featureText, explorationBundle, candidate, qaReport: report };
769
+ }
770
+
771
+ // Stage 1 — Static Validation & Transactional Write
772
+ const staticViolations: string[] = [];
773
+ if (!candidate.fullCodeAuthoring && candidate.symbols.symbols.length > 0) {
774
+ const nl = NamingLinter.lint(candidate.symbols);
775
+ if (!nl.passed) staticViolations.push(...nl.violations.map((v) => v.message));
776
+ }
777
+ const specFiles = candidate.files.filter((f) => /\.(?:spec|test)\.(?:ts|tsx|js|jsx|mjs)$/i.test(f.relativePath));
778
+ for (const spec of specFiles) {
779
+ const aqg = AssertionQualityGate.evaluate(spec.content, featureTitle);
780
+ if (!aqg.passed) staticViolations.push(...aqg.violations);
781
+ }
782
+
783
+ let writtenFiles: string[] = [];
784
+ let modifiedFiles: string[] = [];
785
+ let tscChecked = false;
786
+ let tscPassed: boolean | undefined;
787
+ let discoveryChecked = false;
788
+ let discoveryPassed: boolean | undefined;
789
+
790
+ if (input.write !== false) {
791
+ const writeRes = await CandidateWriter.apply(workspaceRoot, candidate, {
792
+ allowModify: true,
793
+ validateLocalImports: true,
794
+ });
795
+ if (!writeRes.success) {
796
+ staticViolations.push(...writeRes.errors);
797
+ } else {
798
+ writtenFiles = writeRes.written;
799
+ modifiedFiles = writeRes.modified;
800
+ }
801
+ }
802
+
803
+ // Run tsc if tsconfig exists and node_modules tsc exists
804
+ const targetTsconfig = path.join(workspaceRoot, 'tsconfig.json');
805
+ const tscTool = resolveWorkspaceCli(workspaceRoot, 'tsc');
806
+ if (input.write !== false && staticViolations.length === 0 && fsSync.existsSync(targetTsconfig) && tscTool) {
807
+ tscChecked = true;
808
+ const tscRes = await runCommandInWorkspace(tscTool.cmd, [...tscTool.prefixArgs, '-p', targetTsconfig, '--noEmit'], workspaceRoot);
809
+ tscPassed = tscRes.exitCode === 0;
810
+ if (!tscPassed) {
811
+ staticViolations.push(`TypeScript compilation failed: ${(tscRes.stdout + tscRes.stderr).slice(0, 600)}`);
812
+ }
813
+ }
814
+
815
+ const pwTool = resolveWorkspaceCli(workspaceRoot, 'playwright');
816
+ const canRunPlaywright = input.write !== false && Boolean(pwTool);
817
+ if (canRunPlaywright && pwTool && staticViolations.length === 0) {
818
+ discoveryChecked = true;
819
+ const discRes = await runCommandInWorkspace(pwTool.cmd, [...pwTool.prefixArgs, 'test', '--list', ...specFiles.map((s) => s.relativePath)], workspaceRoot);
820
+ discoveryPassed = discRes.exitCode === 0;
821
+ if (!discoveryPassed) {
822
+ staticViolations.push(`Playwright test discovery failed: ${(discRes.stdout + discRes.stderr).slice(0, 600)}`);
823
+ }
824
+ }
825
+
826
+ // Stage 2 — Isolated Execution & Bounded Trace-Guided Repair
827
+ let executed = false;
828
+ let execPassed = false;
829
+ let execExitCode: number | undefined;
830
+ let execDurationMs: number | undefined;
831
+ let execStdout = '';
832
+ const failureClassifications: QaReport['failureClassifications'] = [];
833
+ const repairAttempts: QaReport['repairAttempts'] = [];
834
+ let intentInvariantPreserved = true;
835
+
836
+ if (input.verify !== false && canRunPlaywright && pwTool && staticViolations.length === 0) {
837
+ const maxIter = Math.min(Math.max(input.maxRepairIterations ?? 3, 1), 5);
838
+ const originalSpecContents = new Map(specFiles.map((s) => [s.relativePath, s.content]));
839
+ const started = Date.now();
840
+ executed = true;
841
+
842
+ for (let iter = 0; iter <= maxIter; iter += 1) {
843
+ const runRes = await runCommandInWorkspace(
844
+ pwTool.cmd,
845
+ [...pwTool.prefixArgs, 'test', ...specFiles.map((s) => s.relativePath), '--trace=on', '--reporter=line'],
846
+ workspaceRoot,
847
+ 90_000,
848
+ );
849
+ execExitCode = runRes.exitCode;
850
+ execStdout = (runRes.stdout + '\n' + runRes.stderr).slice(-2000);
851
+ execDurationMs = Date.now() - started;
852
+
853
+ if (execExitCode === 0) {
854
+ execPassed = true;
855
+ break;
856
+ }
857
+
858
+ // Diagnose failure using PIE's Trace/Diagnostic engine when trace.zip exists
859
+ let category = 'UNKNOWN_ERROR';
860
+ let summary = execStdout.slice(0, 300);
861
+ const traceZip = await findFirstTraceZip(workspaceRoot);
862
+ if (traceZip) {
863
+ try {
864
+ const { traceEntryName, networkEntryName, traceEntryNames, networkEntryNames } = ArchiveReader.findTraceFileEntries(traceZip);
865
+ if (traceEntryName) {
866
+ const traceText = await ArchiveReader.readEntriesAsText(traceZip, traceEntryNames ?? [traceEntryName]);
867
+ const networkText = (networkEntryNames ?? (networkEntryName ? [networkEntryName] : [])).length > 0
868
+ ? await ArchiveReader.readEntriesAsText(traceZip, networkEntryNames ?? [networkEntryName!])
869
+ : '';
870
+ const context = TraceParser.assembleDiagnosticContext(traceText.split('\n'), networkText.split('\n'));
871
+ const correlated = EvidenceCorrelator.correlate(context);
872
+ const has500 =
873
+ context.networkEvents.some((n) => n.status >= 500) ||
874
+ correlated.classification.category === 'BACKEND_500' ||
875
+ explorationBundle.network.some((n) => n.status >= 500);
876
+ category = has500 ? 'BACKEND_500' : correlated.classification.category;
877
+ summary = context.failedAction.errorMessage ?? correlated.classification.reason ?? summary;
878
+ }
879
+ } catch {
880
+ // Keep stdout fallback summary
881
+ }
882
+ }
883
+ if (category === 'UNKNOWN_ERROR' && explorationBundle.network.some((n) => n.status >= 500)) {
884
+ category = 'BACKEND_500';
885
+ }
886
+
887
+ const defectOwner: 'automation' | 'application' | 'environment' | 'unknown' =
888
+ category === 'BACKEND_500' ? 'application' : category === 'LOCATOR_DRIFT' ? 'automation' : 'unknown';
889
+ failureClassifications.push({ category, summary, defectOwner });
890
+
891
+ // Never auto-repair application/backend defects or exceed repair budget
892
+ if (defectOwner === 'application' || iter === maxIter || !input.repairCallback) {
893
+ break;
894
+ }
895
+
896
+ const proposedPatch = await input.repairCallback({
897
+ specFiles: candidate.files,
898
+ failureCategory: category,
899
+ errorText: summary,
900
+ iteration: iter + 1,
901
+ });
902
+ if (!proposedPatch) break;
903
+
904
+ // Enforce Assertion Preservation Invariant before accepting repaired spec!
905
+ let invariantOk = true;
906
+ for (const updated of proposedPatch) {
907
+ const orig = originalSpecContents.get(updated.relativePath);
908
+ if (orig && !assertionsPreserved(orig, updated.content)) {
909
+ invariantOk = false;
910
+ }
911
+ }
912
+
913
+ repairAttempts.push({
914
+ iteration: iter + 1,
915
+ category,
916
+ action: invariantOk ? 'apply-repaired-files' : 'rejected-assertion-weakening',
917
+ assertionsPreserved: invariantOk,
918
+ succeeded: false,
919
+ });
920
+
921
+ if (!invariantOk) {
922
+ intentInvariantPreserved = false;
923
+ break;
924
+ }
925
+
926
+ await CandidateWriter.apply(workspaceRoot, { files: proposedPatch.map((f) => ({ ...f, action: 'MODIFY' })) }, { allowModify: true });
927
+ candidate = { ...candidate, files: proposedPatch };
928
+ const recheck = await runCommandInWorkspace(pwTool.cmd, [...pwTool.prefixArgs, 'test', ...specFiles.map((s) => s.relativePath), '--reporter=line'], workspaceRoot, 90_000);
929
+ if (recheck.exitCode === 0) {
930
+ repairAttempts[repairAttempts.length - 1]!.succeeded = true;
931
+ execExitCode = 0;
932
+ execPassed = true;
933
+ break;
934
+ }
935
+ }
936
+ }
937
+
938
+ // Stage 3 — Mutation Falsification (Level C Assertions + Level B API Fault Injection)
939
+ const mutationEntries: MutationResultEntry[] = [];
940
+ let mutationExecuted = false;
941
+
942
+ if (input.runMutations !== false && execPassed && canRunPlaywright && pwTool) {
943
+ mutationExecuted = true;
944
+ for (const spec of specFiles) {
945
+ const targetPath = path.resolve(workspaceRoot, spec.relativePath);
946
+ const original = await fs.readFile(targetPath, 'utf8');
947
+ const levelCMutants = buildAssertionMutants(spec.relativePath, original);
948
+
949
+ for (const mut of levelCMutants) {
950
+ try {
951
+ await fs.writeFile(targetPath, mut.mutatedContent, 'utf8');
952
+ const res = await runCommandInWorkspace(pwTool.cmd, [...pwTool.prefixArgs, 'test', spec.relativePath, '--reporter=line'], workspaceRoot, 60_000);
953
+ mutationEntries.push({
954
+ id: mut.id,
955
+ level: 'LEVEL_C_ASSERTION',
956
+ specFile: spec.relativePath,
957
+ description: mut.description,
958
+ status: res.exitCode !== 0 ? 'KILLED' : 'SURVIVED',
959
+ exitCode: res.exitCode,
960
+ });
961
+ } catch {
962
+ mutationEntries.push({
963
+ id: mut.id,
964
+ level: 'LEVEL_C_ASSERTION',
965
+ specFile: spec.relativePath,
966
+ description: mut.description,
967
+ status: 'INVALID',
968
+ });
969
+ } finally {
970
+ await fs.writeFile(targetPath, original, 'utf8');
971
+ }
972
+ }
973
+
974
+ // Level B: page.route() API fault injection if API calls were observed
975
+ const hasApiEndpoints = explorationBundle.network.some((n) => n.url.includes('/api/'));
976
+ if (hasApiEndpoints) {
977
+ const injected = `test.beforeEach(async ({ page }) => {\n await page.route('**/api/**', (route) => route.fulfill({ status: 500, contentType: 'application/json', body: JSON.stringify({ error: 'PIE_INJECTED_500' }) }));\n});\n`;
978
+ const mutatedApiContent = original.includes('test.describe')
979
+ ? original.replace(/(test\.describe\s*\()/, `${injected}\n$1`)
980
+ : original.replace(/(test\s*\()/, `${injected}\n$1`);
981
+ if (mutatedApiContent !== original) {
982
+ try {
983
+ await fs.writeFile(targetPath, mutatedApiContent, 'utf8');
984
+ const res = await runCommandInWorkspace(pwTool.cmd, [...pwTool.prefixArgs, 'test', spec.relativePath, '--reporter=line'], workspaceRoot, 60_000);
985
+ mutationEntries.push({
986
+ id: 'mut-b-api-500',
987
+ level: 'LEVEL_B_API_FAULT',
988
+ specFile: spec.relativePath,
989
+ description: `Inject HTTP 500 via page.route('**/api/**') in ${spec.relativePath}`,
990
+ status: res.exitCode !== 0 ? 'KILLED' : 'SURVIVED',
991
+ exitCode: res.exitCode,
992
+ });
993
+ } finally {
994
+ await fs.writeFile(targetPath, original, 'utf8');
995
+ }
996
+ }
997
+ }
998
+ }
999
+ }
1000
+
1001
+ const killed = mutationEntries.filter((m) => m.status === 'KILLED').length;
1002
+ const survived = mutationEntries.filter((m) => m.status === 'SURVIVED').length;
1003
+ const invalid = mutationEntries.filter((m) => m.status === 'INVALID').length;
1004
+ const equivalent = mutationEntries.filter((m) => m.status === 'EQUIVALENT').length;
1005
+ const validTotal = killed + survived;
1006
+ const mutationScore = validTotal > 0 ? Number(((killed / validTotal) * 100).toFixed(2)) : null;
1007
+
1008
+ const hasDefects = explorationBundle.findings.some((f) => f.category === 'SPEC_VIOLATION' || f.category === 'RUNTIME_ERROR');
1009
+ const finalStatus: QaReport['status'] = staticViolations.length > 0 || (executed && !execPassed)
1010
+ ? 'VERIFICATION_FAILED'
1011
+ : execPassed && (!hasDefects && (mutationScore === null || mutationScore >= 80))
1012
+ ? 'VERIFIED'
1013
+ : execPassed
1014
+ ? 'VERIFIED_WITH_WARNINGS'
1015
+ : 'GENERATED';
1016
+
1017
+ const qaReportJsonPath = path.join(artifactDir, 'qa-report.json');
1018
+ const qaReportMarkdownPath = path.join(artifactDir, 'qa-report.md');
1019
+
1020
+ const qaReport: QaReport = {
1021
+ reportVersion: '1.0',
1022
+ generatedAt: new Date().toISOString(),
1023
+ status: finalStatus,
1024
+ featureName: featureTitle,
1025
+ topology: explorationBundle.topology,
1026
+ runtimeVerified: explorationBundle.runtimeVerified,
1027
+ requirementCoverage: workflowPlan.requirements.map((r) => {
1028
+ const relatedSc = workflowPlan.scenarios.filter((s) => s.requirementRefs.includes(r.id));
1029
+ const statuses = relatedSc.map((s) => explorationBundle.scenarioStatuses.find((st) => st.scenarioId === s.id)?.status);
1030
+ const covStatus = statuses.some((st) => st === 'BLOCKED' || st === 'DEFECT_DETECTED')
1031
+ ? 'BLOCKED'
1032
+ : statuses.some((st) => st === 'VERIFIED')
1033
+ ? 'COVERED'
1034
+ : 'NOT_COVERED';
1035
+ return {
1036
+ requirementId: r.id,
1037
+ title: r.title,
1038
+ status: covStatus,
1039
+ scenarioIds: relatedSc.map((s) => s.id),
1040
+ };
1041
+ }),
1042
+ scenarioCoverage: workflowPlan.scenarios.map((s) => ({
1043
+ scenarioId: s.id,
1044
+ title: s.title,
1045
+ category: s.category,
1046
+ applicability: s.applicability,
1047
+ explorationStatus: explorationBundle.scenarioStatuses.find((st) => st.scenarioId === s.id)?.status,
1048
+ })),
1049
+ workflowStatuses: explorationBundle.scenarioStatuses,
1050
+ generatedFiles: writtenFiles,
1051
+ modifiedFiles,
1052
+ staticValidation: {
1053
+ passed: staticViolations.length === 0,
1054
+ tscChecked,
1055
+ ...(tscPassed !== undefined ? { tscPassed } : {}),
1056
+ discoveryChecked,
1057
+ ...(discoveryPassed !== undefined ? { discoveryPassed } : {}),
1058
+ violations: staticViolations,
1059
+ },
1060
+ execution: {
1061
+ executed,
1062
+ passed: execPassed,
1063
+ ...(execExitCode !== undefined ? { exitCode: execExitCode } : {}),
1064
+ ...(execDurationMs !== undefined ? { durationMs: execDurationMs } : {}),
1065
+ ...(execStdout ? { stdoutSnippet: execStdout } : {}),
1066
+ },
1067
+ failureClassifications,
1068
+ repairAttempts,
1069
+ intentInvariantPreserved,
1070
+ mutationResults: {
1071
+ executed: mutationExecuted,
1072
+ total: mutationEntries.length,
1073
+ validTotal,
1074
+ killed,
1075
+ survived,
1076
+ invalid,
1077
+ equivalent,
1078
+ mutationScore,
1079
+ entries: mutationEntries,
1080
+ },
1081
+ verificationLayers: {
1082
+ UI: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'UI'),
1083
+ API: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'API'),
1084
+ CLIENT_RELOAD: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'CLIENT_RELOAD'),
1085
+ DB: summarizeLayerStatus(explorationBundle.scenarioStatuses, 'DB'),
1086
+ },
1087
+ findings: explorationBundle.findings,
1088
+ unverifiedBehaviors: explorationBundle.scenarioStatuses.filter((s) => s.status !== 'VERIFIED').map((s) => s.scenarioId),
1089
+ limitations: explorationBundle.limitations,
1090
+ artifacts: {
1091
+ workflowPlanPath,
1092
+ featureFilePath,
1093
+ explorationBundlePath,
1094
+ qaReportJsonPath,
1095
+ qaReportMarkdownPath,
1096
+ },
1097
+ environment: {
1098
+ nodeVersion: process.version,
1099
+ platform: process.platform,
1100
+ workspaceRoot,
1101
+ mcpDependenciesUsed: 'none',
1102
+ },
1103
+ };
1104
+
1105
+ await fs.writeFile(qaReportJsonPath, `${JSON.stringify(qaReport, null, 2)}\n`, 'utf8');
1106
+ await fs.writeFile(qaReportMarkdownPath, formatQaReportMarkdown(qaReport), 'utf8');
1107
+
1108
+ return {
1109
+ workflowPlan,
1110
+ featureText,
1111
+ explorationBundle,
1112
+ candidate,
1113
+ qaReport,
1114
+ };
1115
+ }