playwright-intelligence-engine 2.0.1 → 2.0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +229 -229
- package/dist/cli/index.d.ts.map +1 -1
- package/dist/core/version.d.ts +3 -3
- package/dist/index.js +81 -24
- package/package.json +76 -76
- package/src/authoring/author.ts +81 -81
- package/src/authoring/generation-contract.ts +283 -283
- package/src/authoring/hybrid-author.ts +115 -115
- package/src/authoring/index.ts +7 -7
- package/src/authoring/llm-author.ts +282 -282
- package/src/authoring/naming/name-plan.ts +28 -28
- package/src/authoring/naming/name-planner.ts +339 -339
- package/src/authoring/naming/naming-linter.ts +292 -292
- package/src/cli/index.ts +1238 -1172
- package/src/collectors/git-diff-collector.ts +190 -190
- package/src/core/version.ts +25 -25
- package/src/diagnostics/diagnostic-report.ts +204 -204
- package/src/diagnostics/failure-classifier.ts +701 -701
- package/src/evaluation/repository-runner.ts +197 -197
- package/src/gate/quality.ts +101 -101
- package/src/generator/architecture-generator.ts +582 -582
- package/src/generator/assertion-quality-gate.ts +88 -88
- package/src/generator/candidate-writer.ts +288 -288
- package/src/generator/test-generator.ts +52 -52
- package/src/index.ts +107 -107
- package/src/intelligence/evidence-bundle.ts +300 -300
- package/src/intelligence/generation-agent.ts +561 -561
- package/src/intelligence/generation-verifier.ts +1115 -1115
- package/src/intelligence/locator-disambiguation-engine.ts +331 -331
- package/src/intelligence/project-intelligence.ts +140 -140
- package/src/intelligence/repository-profile.ts +796 -796
- package/src/intelligence/trace-proposal-workflow.ts +318 -318
- package/src/intelligence/workflow-planner.ts +966 -966
- package/src/mcp/server.ts +970 -970
- package/src/perception/app-model.ts +32 -32
- package/src/perception/crawler.ts +263 -263
- package/src/perception/index.ts +3 -3
- package/src/repair/assertion-hasher.ts +386 -386
- package/src/repair/diff-allowlist.ts +612 -612
- package/src/repair/intent-validator.ts +761 -761
- package/src/repair/repair-planner.ts +467 -467
- package/src/repair/statement-patcher.ts +544 -544
- package/src/reproduce/interactive-workflow-runner.ts +828 -828
- package/src/sandbox/execution-runner.ts +408 -408
- package/src/selector/abstention-policy.ts +175 -175
- package/src/selector/candidate-extractor.ts +343 -343
- package/src/selector/repair-candidate-ranker.ts +92 -92
- package/src/selector/selector-scorer.ts +344 -344
|
@@ -1,828 +1,828 @@
|
|
|
1
|
-
import fs from 'node:fs/promises';
|
|
2
|
-
import path from 'node:path';
|
|
3
|
-
import { EvidenceRedactor } from '../evidence/evidence-redactor.js';
|
|
4
|
-
import { NetworkCollector, type RawNetworkEvent } from '../collectors/network-collector.js';
|
|
5
|
-
import { LiveLocatorVerifier } from '../selector/live-locator-verifier.js';
|
|
6
|
-
import type { LocatorPlan } from '../selector/candidate-extractor.js';
|
|
7
|
-
import {
|
|
8
|
-
ExplorationActionRecord,
|
|
9
|
-
ExplorationContextBundle,
|
|
10
|
-
ExplorationFinding,
|
|
11
|
-
ExplorationNetworkRecord,
|
|
12
|
-
loadExplorationContextBundle,
|
|
13
|
-
redactExplorationBundle,
|
|
14
|
-
sanitizeUrlSecrets,
|
|
15
|
-
saveExplorationContextBundle,
|
|
16
|
-
ScenarioExplorationStatus,
|
|
17
|
-
VerificationLayerStatus,
|
|
18
|
-
VerifiedLocatorEvidence,
|
|
19
|
-
} from '../intelligence/evidence-bundle.js';
|
|
20
|
-
|
|
21
|
-
function buildExplorationContextBundleUrlSanitized(baseUrl?: string): string | undefined {
|
|
22
|
-
if (!baseUrl) return undefined;
|
|
23
|
-
return sanitizeUrlSecrets(baseUrl);
|
|
24
|
-
}
|
|
25
|
-
import {
|
|
26
|
-
buildBrownfieldRepositoryGraph,
|
|
27
|
-
type BrownfieldRepositoryGraph,
|
|
28
|
-
} from '../intelligence/repository-profile.js';
|
|
29
|
-
import type {
|
|
30
|
-
GenerationWorkflowPlan,
|
|
31
|
-
PlannedWorkflowScenario,
|
|
32
|
-
} from '../intelligence/workflow-planner.js';
|
|
33
|
-
|
|
34
|
-
export interface ControlledLoginConfig {
|
|
35
|
-
loginUrl: string;
|
|
36
|
-
usernameSelector?: string | undefined;
|
|
37
|
-
passwordSelector?: string | undefined;
|
|
38
|
-
submitSelector?: string | undefined;
|
|
39
|
-
username?: string | undefined;
|
|
40
|
-
password?: string | undefined;
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
export interface DbVerificationResult {
|
|
44
|
-
status: VerificationLayerStatus;
|
|
45
|
-
summary: string;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export interface InteractiveExplorerOptions {
|
|
49
|
-
baseUrl?: string | undefined;
|
|
50
|
-
workspaceRoot?: string | undefined;
|
|
51
|
-
workflowPlan: GenerationWorkflowPlan;
|
|
52
|
-
checkpointPath?: string | undefined;
|
|
53
|
-
storageStatePath?: string | undefined;
|
|
54
|
-
loginConfig?: ControlledLoginConfig | undefined;
|
|
55
|
-
allowMutatingActions?: boolean | undefined;
|
|
56
|
-
allowDestructiveActions?: boolean | undefined;
|
|
57
|
-
designatedTestEnvironment?: boolean | undefined;
|
|
58
|
-
maxActionsPerScenario?: number | undefined;
|
|
59
|
-
timeoutMs?: number | undefined;
|
|
60
|
-
browser?: any;
|
|
61
|
-
page?: any;
|
|
62
|
-
dbVerifier?: ((scenario: PlannedWorkflowScenario, actions: ExplorationActionRecord[]) => Promise<DbVerificationResult>) | undefined;
|
|
63
|
-
repositoryGraph?: BrownfieldRepositoryGraph | undefined;
|
|
64
|
-
onCheckpoint?: ((bundle: ExplorationContextBundle) => void | Promise<void>) | undefined;
|
|
65
|
-
simulateCrashAfterScenarioIndex?: number | undefined;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
const DESTRUCTIVE_TEXT_RE = /\b(delete\s+account|destroy\s+database|drop\s+table|charge\s+card|complete\s+real\s+payment|wipe\s+all)\b/i;
|
|
69
|
-
|
|
70
|
-
function isLocalhostUrl(url?: string): boolean {
|
|
71
|
-
if (!url) return false;
|
|
72
|
-
return /https?:\/\/(?:localhost|127\.0\.0\.1|\[::1\])(?::\d+)?/i.test(url);
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
interface RawDomElementInfo {
|
|
76
|
-
tag: string;
|
|
77
|
-
role: string;
|
|
78
|
-
accessibleName: string;
|
|
79
|
-
placeholder?: string | undefined;
|
|
80
|
-
label?: string | undefined;
|
|
81
|
-
testId?: string | undefined;
|
|
82
|
-
type?: string | undefined;
|
|
83
|
-
visible: boolean;
|
|
84
|
-
frameSelector?: string | undefined;
|
|
85
|
-
}
|
|
86
|
-
|
|
87
|
-
function buildLocatorPlanFromDom(el: RawDomElementInfo): {
|
|
88
|
-
strategy: VerifiedLocatorEvidence['strategy'];
|
|
89
|
-
expression: string;
|
|
90
|
-
plan: LocatorPlan;
|
|
91
|
-
} {
|
|
92
|
-
if (el.testId) {
|
|
93
|
-
return {
|
|
94
|
-
strategy: 'testId',
|
|
95
|
-
expression: `page.getByTestId(${JSON.stringify(el.testId)})`,
|
|
96
|
-
plan: { kind: 'getByTestId', testId: el.testId },
|
|
97
|
-
};
|
|
98
|
-
}
|
|
99
|
-
if (el.placeholder) {
|
|
100
|
-
return {
|
|
101
|
-
strategy: 'placeholder',
|
|
102
|
-
expression: `page.getByPlaceholder(${JSON.stringify(el.placeholder)})`,
|
|
103
|
-
plan: { kind: 'getByPlaceholder', placeholder: el.placeholder },
|
|
104
|
-
};
|
|
105
|
-
}
|
|
106
|
-
if (el.label && (el.tag === 'input' || el.tag === 'textarea' || el.tag === 'select')) {
|
|
107
|
-
return {
|
|
108
|
-
strategy: 'label',
|
|
109
|
-
expression: `page.getByLabel(${JSON.stringify(el.label)})`,
|
|
110
|
-
plan: { kind: 'getByLabel', label: el.label },
|
|
111
|
-
};
|
|
112
|
-
}
|
|
113
|
-
if (el.role && el.accessibleName) {
|
|
114
|
-
return {
|
|
115
|
-
strategy: 'role',
|
|
116
|
-
expression: `page.getByRole(${JSON.stringify(el.role)}, { name: ${JSON.stringify(el.accessibleName)} })`,
|
|
117
|
-
plan: { kind: 'getByRole', role: el.role, options: { name: el.accessibleName } },
|
|
118
|
-
};
|
|
119
|
-
}
|
|
120
|
-
return {
|
|
121
|
-
strategy: 'css',
|
|
122
|
-
expression: `page.locator(${JSON.stringify(el.tag)})`,
|
|
123
|
-
plan: { kind: 'locator', selector: el.tag },
|
|
124
|
-
};
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
async function captureVisibleTextSnapshot(page: any): Promise<{ summary: string; lines: string[] }> {
|
|
128
|
-
try {
|
|
129
|
-
const raw = await page.evaluate(() => {
|
|
130
|
-
const doc = (globalThis as any).document;
|
|
131
|
-
const bodyText = (doc?.body?.innerText ?? '').trim();
|
|
132
|
-
const lines = bodyText.split(/\r?\n/).map((l: string) => l.trim()).filter(Boolean).slice(0, 60);
|
|
133
|
-
return {
|
|
134
|
-
summary: lines.join(' | ').slice(0, 1200),
|
|
135
|
-
lines,
|
|
136
|
-
};
|
|
137
|
-
});
|
|
138
|
-
return raw;
|
|
139
|
-
} catch {
|
|
140
|
-
return { summary: '', lines: [] };
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
|
|
144
|
-
async function collectInteractiveControls(page: any): Promise<RawDomElementInfo[]> {
|
|
145
|
-
try {
|
|
146
|
-
const raw = await page.$$eval(
|
|
147
|
-
'input, textarea, select, button, a[href], [role="button"], [role="textbox"], [role="checkbox"], [role="status"], [role="alert"], h1, h2, h3',
|
|
148
|
-
(nodes: any[]) =>
|
|
149
|
-
nodes.slice(0, 80).map((el) => {
|
|
150
|
-
const tag = el.tagName.toLowerCase();
|
|
151
|
-
const type = el.getAttribute('type') || undefined;
|
|
152
|
-
let role = el.getAttribute('role') || '';
|
|
153
|
-
if (!role) {
|
|
154
|
-
if (tag === 'button' || (tag === 'input' && (type === 'submit' || type === 'button'))) role = 'button';
|
|
155
|
-
else if (tag === 'input' && type === 'checkbox') role = 'checkbox';
|
|
156
|
-
else if (tag === 'input' || tag === 'textarea') role = 'textbox';
|
|
157
|
-
else if (tag === 'select') role = 'combobox';
|
|
158
|
-
else if (tag === 'a') role = 'link';
|
|
159
|
-
else if (/^h[1-3]$/.test(tag)) role = 'heading';
|
|
160
|
-
}
|
|
161
|
-
const placeholder = el.getAttribute('placeholder') || undefined;
|
|
162
|
-
const label = el.getAttribute('aria-label') || el.getAttribute('name') || undefined;
|
|
163
|
-
const testId = el.getAttribute('data-testid') || el.getAttribute('data-test') || undefined;
|
|
164
|
-
const accessibleName = (el.getAttribute('aria-label') || el.innerText || placeholder || '').trim().slice(0, 120);
|
|
165
|
-
const visible = Boolean(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
|
|
166
|
-
return { tag, role, accessibleName, placeholder, label, testId, type, visible };
|
|
167
|
-
}),
|
|
168
|
-
);
|
|
169
|
-
return raw.filter((r: RawDomElementInfo) => r.visible);
|
|
170
|
-
} catch {
|
|
171
|
-
return [];
|
|
172
|
-
}
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
function diffLines(before: string[], after: string[]): string[] {
|
|
176
|
-
const beforeSet = new Set(before);
|
|
177
|
-
const afterSet = new Set(after);
|
|
178
|
-
const diffs: string[] = [];
|
|
179
|
-
for (const line of after) {
|
|
180
|
-
if (!beforeSet.has(line)) diffs.push(`+ ${line}`);
|
|
181
|
-
}
|
|
182
|
-
for (const line of before) {
|
|
183
|
-
if (!afterSet.has(line)) diffs.push(`- ${line}`);
|
|
184
|
-
}
|
|
185
|
-
return diffs.slice(0, 40);
|
|
186
|
-
}
|
|
187
|
-
|
|
188
|
-
export class InteractiveWorkflowRunner {
|
|
189
|
-
/**
|
|
190
|
-
* Executes workflow exploration & manual verification in a single native Playwright session
|
|
191
|
-
* with action-level checkpoints, passive defect classification, and redaction.
|
|
192
|
-
*/
|
|
193
|
-
public static async run(options: InteractiveExplorerOptions): Promise<ExplorationContextBundle> {
|
|
194
|
-
const repoGraph = options.repositoryGraph ?? (
|
|
195
|
-
options.workspaceRoot ? await buildBrownfieldRepositoryGraph(options.workspaceRoot) : undefined
|
|
196
|
-
);
|
|
197
|
-
const topology: ExplorationContextBundle['topology'] = !options.baseUrl
|
|
198
|
-
? 'source-only'
|
|
199
|
-
: repoGraph?.topology === 'monorepo'
|
|
200
|
-
? 'monorepo'
|
|
201
|
-
: 'standalone';
|
|
202
|
-
|
|
203
|
-
const bundleId = `explore-${Date.now()}`;
|
|
204
|
-
const networkCollector = new NetworkCollector();
|
|
205
|
-
const rawNetworkEvents: RawNetworkEvent[] = [];
|
|
206
|
-
const networkRecords: ExplorationNetworkRecord[] = [];
|
|
207
|
-
const consoleErrors: string[] = [];
|
|
208
|
-
const verifiedLocators: VerifiedLocatorEvidence[] = [];
|
|
209
|
-
const seenLocatorExpr = new Set<string>();
|
|
210
|
-
const actionRecords: ExplorationActionRecord[] = [];
|
|
211
|
-
const findings: ExplorationFinding[] = [];
|
|
212
|
-
const scenarioStatuses: ScenarioExplorationStatus[] = [];
|
|
213
|
-
const persistenceObservations: ExplorationContextBundle['persistenceObservations'] = [];
|
|
214
|
-
const routesMap = new Map<string, { route: string; title?: string; discoveredAt: string; locatorIds: string[] }>();
|
|
215
|
-
const limitations: string[] = [];
|
|
216
|
-
const evidenceIndex: ExplorationContextBundle['evidenceIndex'] = [];
|
|
217
|
-
|
|
218
|
-
const authMethod: ExplorationContextBundle['authentication']['method'] = options.storageStatePath
|
|
219
|
-
? 'storage-state'
|
|
220
|
-
: options.loginConfig
|
|
221
|
-
? 'login-flow'
|
|
222
|
-
: (repoGraph?.storageStatePaths.length ?? 0) > 0
|
|
223
|
-
? 'existing-fixture'
|
|
224
|
-
: 'none';
|
|
225
|
-
let authenticated = authMethod === 'storage-state';
|
|
226
|
-
|
|
227
|
-
const buildSnapshot = (runtimeVerified: boolean): ExplorationContextBundle => {
|
|
228
|
-
const sanitizedBase = buildExplorationContextBundleUrlSanitized(options.baseUrl);
|
|
229
|
-
const bundle: ExplorationContextBundle = {
|
|
230
|
-
bundleVersion: '1.0',
|
|
231
|
-
bundleId,
|
|
232
|
-
createdAt: new Date().toISOString(),
|
|
233
|
-
topology,
|
|
234
|
-
runtimeVerified,
|
|
235
|
-
...(sanitizedBase ? { baseUrl: sanitizedBase } : {}),
|
|
236
|
-
authentication: {
|
|
237
|
-
configured: authMethod !== 'none',
|
|
238
|
-
method: authMethod,
|
|
239
|
-
authenticated,
|
|
240
|
-
secretsExposed: false,
|
|
241
|
-
},
|
|
242
|
-
requirements: {
|
|
243
|
-
featureName: options.workflowPlan.featureName,
|
|
244
|
-
requirementIds: options.workflowPlan.requirements.map((r) => r.id),
|
|
245
|
-
scenarioIds: options.workflowPlan.scenarios.map((s) => s.id),
|
|
246
|
-
workflowPlanHash: options.workflowPlan.planHash,
|
|
247
|
-
},
|
|
248
|
-
routes: [...routesMap.values()],
|
|
249
|
-
locators: verifiedLocators,
|
|
250
|
-
actions: actionRecords,
|
|
251
|
-
network: networkRecords,
|
|
252
|
-
consoleErrors: [...new Set(consoleErrors)],
|
|
253
|
-
persistenceObservations,
|
|
254
|
-
scenarioStatuses,
|
|
255
|
-
findings,
|
|
256
|
-
...(repoGraph ? { repositoryGraph: repoGraph } : {}),
|
|
257
|
-
evidenceIndex,
|
|
258
|
-
limitations: [...new Set(limitations)],
|
|
259
|
-
redactionComplete: true,
|
|
260
|
-
};
|
|
261
|
-
return redactExplorationBundle(bundle);
|
|
262
|
-
};
|
|
263
|
-
|
|
264
|
-
const persistCheckpoint = async (runtimeVerified: boolean) => {
|
|
265
|
-
const snapshot = buildSnapshot(runtimeVerified);
|
|
266
|
-
if (options.checkpointPath) {
|
|
267
|
-
await saveExplorationContextBundle(options.checkpointPath, snapshot);
|
|
268
|
-
}
|
|
269
|
-
if (options.onCheckpoint) {
|
|
270
|
-
await options.onCheckpoint(snapshot);
|
|
271
|
-
}
|
|
272
|
-
};
|
|
273
|
-
|
|
274
|
-
// Source-only offline mode when no baseUrl is provided
|
|
275
|
-
if (!options.baseUrl) {
|
|
276
|
-
limitations.push('No baseUrl supplied; exploration ran in NOT_RUNTIME_VERIFIED source-only mode.');
|
|
277
|
-
for (const sc of options.workflowPlan.scenarios) {
|
|
278
|
-
scenarioStatuses.push({
|
|
279
|
-
scenarioId: sc.id,
|
|
280
|
-
status: 'NOT_VERIFIED',
|
|
281
|
-
verificationLayers: {
|
|
282
|
-
UI: 'NOT_VERIFIED',
|
|
283
|
-
API: 'NOT_VERIFIED',
|
|
284
|
-
CLIENT_RELOAD: 'NOT_VERIFIED',
|
|
285
|
-
DB: 'NOT_APPLICABLE',
|
|
286
|
-
},
|
|
287
|
-
reason: 'No running application baseUrl supplied.',
|
|
288
|
-
evidenceIds: [],
|
|
289
|
-
});
|
|
290
|
-
}
|
|
291
|
-
const bundle = buildSnapshot(false);
|
|
292
|
-
await persistCheckpoint(false);
|
|
293
|
-
return bundle;
|
|
294
|
-
}
|
|
295
|
-
|
|
296
|
-
let browserToClose: any = null;
|
|
297
|
-
let page = options.page;
|
|
298
|
-
|
|
299
|
-
try {
|
|
300
|
-
if (!page) {
|
|
301
|
-
if (options.browser) {
|
|
302
|
-
const context = await options.browser.newContext({
|
|
303
|
-
...(options.storageStatePath ? { storageState: options.storageStatePath } : {}),
|
|
304
|
-
});
|
|
305
|
-
page = await context.newPage();
|
|
306
|
-
} else {
|
|
307
|
-
const { chromium } = await import('@playwright/test');
|
|
308
|
-
browserToClose = await chromium.launch({ headless: true });
|
|
309
|
-
const context = await browserToClose.newContext({
|
|
310
|
-
...(options.storageStatePath ? { storageState: options.storageStatePath } : {}),
|
|
311
|
-
});
|
|
312
|
-
page = await context.newPage();
|
|
313
|
-
}
|
|
314
|
-
}
|
|
315
|
-
|
|
316
|
-
const timeoutMs = options.timeoutMs ?? 15_000;
|
|
317
|
-
let currentScenarioId: string | undefined;
|
|
318
|
-
|
|
319
|
-
if (typeof page.on === 'function') {
|
|
320
|
-
page.on('response', (res: any) => {
|
|
321
|
-
try {
|
|
322
|
-
const req = typeof res.request === 'function' ? res.request() : undefined;
|
|
323
|
-
const evt: RawNetworkEvent = {
|
|
324
|
-
url: typeof res.url === 'function' ? res.url() : '',
|
|
325
|
-
method: req && typeof req.method === 'function' ? req.method() : 'GET',
|
|
326
|
-
status: typeof res.status === 'function' ? res.status() : 200,
|
|
327
|
-
resourceType: req && typeof req.resourceType === 'function' ? req.resourceType() : undefined,
|
|
328
|
-
timestamp: Date.now(),
|
|
329
|
-
};
|
|
330
|
-
rawNetworkEvents.push(evt);
|
|
331
|
-
const recId = `net-${networkRecords.length + 1}`;
|
|
332
|
-
const isError = evt.status >= 400;
|
|
333
|
-
networkRecords.push({
|
|
334
|
-
id: recId,
|
|
335
|
-
...(currentScenarioId ? { scenarioId: currentScenarioId } : {}),
|
|
336
|
-
method: evt.method,
|
|
337
|
-
url: EvidenceRedactor.redactText(evt.url).redactedText,
|
|
338
|
-
status: evt.status,
|
|
339
|
-
isError,
|
|
340
|
-
...(evt.resourceType ? { resourceType: evt.resourceType } : {}),
|
|
341
|
-
timestamp: new Date(evt.timestamp!).toISOString(),
|
|
342
|
-
});
|
|
343
|
-
} catch {}
|
|
344
|
-
});
|
|
345
|
-
page.on('pageerror', (err: any) => {
|
|
346
|
-
consoleErrors.push(EvidenceRedactor.redactText(err?.message ?? String(err)).redactedText);
|
|
347
|
-
});
|
|
348
|
-
page.on('console', (msg: any) => {
|
|
349
|
-
if (typeof msg.type === 'function' && msg.type() === 'error') {
|
|
350
|
-
consoleErrors.push(EvidenceRedactor.redactText(typeof msg.text === 'function' ? msg.text() : String(msg)).redactedText);
|
|
351
|
-
}
|
|
352
|
-
});
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
// Optional controlled login (credentials NEVER placed in evidence)
|
|
356
|
-
if (options.loginConfig) {
|
|
357
|
-
await page.goto(options.loginConfig.loginUrl, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
|
|
358
|
-
if (options.loginConfig.usernameSelector && options.loginConfig.username) {
|
|
359
|
-
await page.locator(options.loginConfig.usernameSelector).fill(options.loginConfig.username);
|
|
360
|
-
}
|
|
361
|
-
if (options.loginConfig.passwordSelector && options.loginConfig.password) {
|
|
362
|
-
await page.locator(options.loginConfig.passwordSelector).fill(options.loginConfig.password);
|
|
363
|
-
}
|
|
364
|
-
if (options.loginConfig.submitSelector) {
|
|
365
|
-
await page.locator(options.loginConfig.submitSelector).click();
|
|
366
|
-
}
|
|
367
|
-
authenticated = true;
|
|
368
|
-
}
|
|
369
|
-
|
|
370
|
-
const allowDestructive = Boolean(
|
|
371
|
-
options.allowDestructiveActions && (options.designatedTestEnvironment || isLocalhostUrl(options.baseUrl)),
|
|
372
|
-
);
|
|
373
|
-
const allowMutating = options.allowMutatingActions ?? isLocalhostUrl(options.baseUrl);
|
|
374
|
-
let lastCreatedValue: string | undefined;
|
|
375
|
-
|
|
376
|
-
// Multi-route discovery sweep: inspect same-origin links & static routes once before scenarios
|
|
377
|
-
try {
|
|
378
|
-
await page.goto(options.baseUrl, { waitUntil: 'networkidle', timeout: Math.min(timeoutMs, 6_000) }).catch(() =>
|
|
379
|
-
page.goto(options.baseUrl, { waitUntil: 'domcontentloaded', timeout: timeoutMs }),
|
|
380
|
-
);
|
|
381
|
-
const origin = new URL(options.baseUrl).origin;
|
|
382
|
-
const linkHrefs: string[] = await page.$$eval('a[href]', (anchors: any[]) =>
|
|
383
|
-
anchors.map((a) => String(a.getAttribute('href') || '')).filter(Boolean),
|
|
384
|
-
).catch(() => []);
|
|
385
|
-
const candidateRoutes = new Set<string>([options.baseUrl]);
|
|
386
|
-
for (const href of linkHrefs) {
|
|
387
|
-
if (href.startsWith('#') || /^(?:mailto:|tel:|javascript:)/i.test(href)) continue;
|
|
388
|
-
if (/logout|signout|delete|destroy|remove/i.test(href)) continue;
|
|
389
|
-
try {
|
|
390
|
-
const resolved = new URL(href, options.baseUrl);
|
|
391
|
-
if (resolved.origin === origin) candidateRoutes.add(resolved.href);
|
|
392
|
-
} catch {}
|
|
393
|
-
}
|
|
394
|
-
for (const hintRoute of repoGraph?.sourceHints.routes ?? []) {
|
|
395
|
-
if (!hintRoute.startsWith('/') || hintRoute.includes(':') || /logout|signout|delete/i.test(hintRoute)) continue;
|
|
396
|
-
try {
|
|
397
|
-
candidateRoutes.add(new URL(hintRoute, origin).href);
|
|
398
|
-
} catch {}
|
|
399
|
-
}
|
|
400
|
-
const routesToSweep = [...candidateRoutes].slice(0, 5);
|
|
401
|
-
for (const routeUrl of routesToSweep) {
|
|
402
|
-
await page.goto(routeUrl, { waitUntil: 'networkidle', timeout: Math.min(timeoutMs, 4_000) }).catch(() =>
|
|
403
|
-
page.goto(routeUrl, { waitUntil: 'domcontentloaded', timeout: timeoutMs }),
|
|
404
|
-
);
|
|
405
|
-
const sweepPathname = (() => {
|
|
406
|
-
try {
|
|
407
|
-
return new URL(page.url()).pathname || '/';
|
|
408
|
-
} catch {
|
|
409
|
-
return '/';
|
|
410
|
-
}
|
|
411
|
-
})();
|
|
412
|
-
const sweepTitle = typeof page.title === 'function' ? await page.title().catch(() => undefined) : undefined;
|
|
413
|
-
if (!routesMap.has(sweepPathname)) {
|
|
414
|
-
routesMap.set(sweepPathname, {
|
|
415
|
-
route: sweepPathname,
|
|
416
|
-
...(sweepTitle ? { title: sweepTitle } : {}),
|
|
417
|
-
discoveredAt: new Date().toISOString(),
|
|
418
|
-
locatorIds: [],
|
|
419
|
-
});
|
|
420
|
-
}
|
|
421
|
-
const sweepControls = await collectInteractiveControls(page);
|
|
422
|
-
for (const ctrl of sweepControls) {
|
|
423
|
-
const built = buildLocatorPlanFromDom(ctrl);
|
|
424
|
-
if (seenLocatorExpr.has(`${sweepPathname}::${built.expression}`)) continue;
|
|
425
|
-
const ver = await LiveLocatorVerifier.verify(page, built.plan);
|
|
426
|
-
const locId = `loc-${verifiedLocators.length + 1}`;
|
|
427
|
-
seenLocatorExpr.add(`${sweepPathname}::${built.expression}`);
|
|
428
|
-
verifiedLocators.push({
|
|
429
|
-
id: locId,
|
|
430
|
-
route: sweepPathname,
|
|
431
|
-
strategy: built.strategy,
|
|
432
|
-
expression: built.expression,
|
|
433
|
-
...(ctrl.role ? { role: ctrl.role } : {}),
|
|
434
|
-
...(ctrl.accessibleName ? { accessibleName: ctrl.accessibleName } : {}),
|
|
435
|
-
...(ctrl.label ? { label: ctrl.label } : {}),
|
|
436
|
-
...(ctrl.placeholder ? { placeholder: ctrl.placeholder } : {}),
|
|
437
|
-
...(ctrl.testId ? { testId: ctrl.testId } : {}),
|
|
438
|
-
matchCount: ver.count,
|
|
439
|
-
visible: ver.isVisible,
|
|
440
|
-
enabled: ver.isEnabled,
|
|
441
|
-
confidence: ver.isUnique ? 0.95 : 0.5,
|
|
442
|
-
timestamp: new Date().toISOString(),
|
|
443
|
-
});
|
|
444
|
-
routesMap.get(sweepPathname)!.locatorIds.push(locId);
|
|
445
|
-
}
|
|
446
|
-
}
|
|
447
|
-
} catch {
|
|
448
|
-
// Ignore non-fatal route sweep errors; scenario loop continues directly on baseUrl
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
for (let scIdx = 0; scIdx < options.workflowPlan.scenarios.length; scIdx += 1) {
|
|
452
|
-
const scenario = options.workflowPlan.scenarios[scIdx]!;
|
|
453
|
-
currentScenarioId = scenario.id;
|
|
454
|
-
|
|
455
|
-
if (options.simulateCrashAfterScenarioIndex !== undefined && scIdx >= options.simulateCrashAfterScenarioIndex) {
|
|
456
|
-
throw new Error(`Simulated exploration interruption before scenario '${scenario.id}'`);
|
|
457
|
-
}
|
|
458
|
-
|
|
459
|
-
if (scenario.applicability === 'needs_clarification') {
|
|
460
|
-
scenarioStatuses.push({
|
|
461
|
-
scenarioId: scenario.id,
|
|
462
|
-
status: 'NOT_VERIFIED',
|
|
463
|
-
verificationLayers: { UI: 'NOT_VERIFIED', API: 'NOT_APPLICABLE', CLIENT_RELOAD: 'NOT_APPLICABLE', DB: 'NOT_APPLICABLE' },
|
|
464
|
-
reason: 'Scenario marked needs_clarification in workflow plan.',
|
|
465
|
-
evidenceIds: [],
|
|
466
|
-
});
|
|
467
|
-
continue;
|
|
468
|
-
}
|
|
469
|
-
|
|
470
|
-
const netStartIdx = networkRecords.length;
|
|
471
|
-
const errStartIdx = consoleErrors.length;
|
|
472
|
-
const scEvidenceIds: string[] = [];
|
|
473
|
-
|
|
474
|
-
// Navigate to baseUrl for deterministic isolation (skip reload scenarios so state stays)
|
|
475
|
-
if (scenario.category !== 'persistence') {
|
|
476
|
-
await page.goto(options.baseUrl, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
const routePathname = (() => {
|
|
480
|
-
try {
|
|
481
|
-
return new URL(page.url()).pathname || '/';
|
|
482
|
-
} catch {
|
|
483
|
-
return '/';
|
|
484
|
-
}
|
|
485
|
-
})();
|
|
486
|
-
const pageTitle = typeof page.title === 'function' ? await page.title().catch(() => undefined) : undefined;
|
|
487
|
-
if (!routesMap.has(routePathname)) {
|
|
488
|
-
routesMap.set(routePathname, {
|
|
489
|
-
route: routePathname,
|
|
490
|
-
...(pageTitle ? { title: pageTitle } : {}),
|
|
491
|
-
discoveredAt: new Date().toISOString(),
|
|
492
|
-
locatorIds: [],
|
|
493
|
-
});
|
|
494
|
-
}
|
|
495
|
-
|
|
496
|
-
const controls = await collectInteractiveControls(page);
|
|
497
|
-
for (const ctrl of controls) {
|
|
498
|
-
const built = buildLocatorPlanFromDom(ctrl);
|
|
499
|
-
if (seenLocatorExpr.has(`${routePathname}::${built.expression}`)) continue;
|
|
500
|
-
const ver = await LiveLocatorVerifier.verify(page, built.plan);
|
|
501
|
-
const locId = `loc-${verifiedLocators.length + 1}`;
|
|
502
|
-
seenLocatorExpr.add(`${routePathname}::${built.expression}`);
|
|
503
|
-
verifiedLocators.push({
|
|
504
|
-
id: locId,
|
|
505
|
-
route: routePathname,
|
|
506
|
-
strategy: built.strategy,
|
|
507
|
-
expression: built.expression,
|
|
508
|
-
...(ctrl.role ? { role: ctrl.role } : {}),
|
|
509
|
-
...(ctrl.accessibleName ? { accessibleName: ctrl.accessibleName } : {}),
|
|
510
|
-
...(ctrl.label ? { label: ctrl.label } : {}),
|
|
511
|
-
...(ctrl.placeholder ? { placeholder: ctrl.placeholder } : {}),
|
|
512
|
-
...(ctrl.testId ? { testId: ctrl.testId } : {}),
|
|
513
|
-
matchCount: ver.count,
|
|
514
|
-
visible: ver.isVisible,
|
|
515
|
-
enabled: ver.isEnabled,
|
|
516
|
-
confidence: ver.isUnique ? 0.95 : 0.5,
|
|
517
|
-
timestamp: new Date().toISOString(),
|
|
518
|
-
});
|
|
519
|
-
routesMap.get(routePathname)!.locatorIds.push(locId);
|
|
520
|
-
scEvidenceIds.push(locId);
|
|
521
|
-
evidenceIndex.push({
|
|
522
|
-
id: locId,
|
|
523
|
-
kind: 'verified-locator',
|
|
524
|
-
source: 'playwright-live-dom',
|
|
525
|
-
timestamp: new Date().toISOString(),
|
|
526
|
-
confidence: ver.isUnique ? 0.95 : 0.5,
|
|
527
|
-
scenarioIds: [scenario.id],
|
|
528
|
-
description: `${built.expression} (count=${ver.count})`,
|
|
529
|
-
});
|
|
530
|
-
}
|
|
531
|
-
|
|
532
|
-
// Check destructive action guard against scenario text
|
|
533
|
-
const combinedScenarioText = `${scenario.title} ${scenario.steps.map((s) => s.text).join(' ')}`;
|
|
534
|
-
if (DESTRUCTIVE_TEXT_RE.test(combinedScenarioText) && !allowDestructive) {
|
|
535
|
-
actionRecords.push({
|
|
536
|
-
id: `act-${actionRecords.length + 1}`,
|
|
537
|
-
scenarioId: scenario.id,
|
|
538
|
-
stepIndex: 0,
|
|
539
|
-
action: 'destructive-policy-check',
|
|
540
|
-
routeBefore: routePathname,
|
|
541
|
-
routeAfter: routePathname,
|
|
542
|
-
domBeforeSummary: '',
|
|
543
|
-
domAfterSummary: '',
|
|
544
|
-
domDiff: [],
|
|
545
|
-
networkIds: [],
|
|
546
|
-
consoleErrors: [],
|
|
547
|
-
status: 'BLOCKED_BY_POLICY',
|
|
548
|
-
timestamp: new Date().toISOString(),
|
|
549
|
-
});
|
|
550
|
-
scenarioStatuses.push({
|
|
551
|
-
scenarioId: scenario.id,
|
|
552
|
-
status: 'BLOCKED',
|
|
553
|
-
verificationLayers: { UI: 'NOT_VERIFIED', API: 'NOT_VERIFIED', CLIENT_RELOAD: 'NOT_APPLICABLE', DB: 'NOT_APPLICABLE' },
|
|
554
|
-
reason: 'Destructive operation blocked by safety policy (requires designatedTestEnvironment & allowDestructiveActions).',
|
|
555
|
-
evidenceIds: scEvidenceIds,
|
|
556
|
-
});
|
|
557
|
-
await persistCheckpoint(true);
|
|
558
|
-
continue;
|
|
559
|
-
}
|
|
560
|
-
|
|
561
|
-
const beforeSnap = await captureVisibleTextSnapshot(page);
|
|
562
|
-
let uiStatus: VerificationLayerStatus = 'NOT_VERIFIED';
|
|
563
|
-
let clientReloadStatus: VerificationLayerStatus = 'NOT_APPLICABLE';
|
|
564
|
-
|
|
565
|
-
const textInputLoc = verifiedLocators.find(
|
|
566
|
-
(l) => l.route === routePathname && (l.strategy === 'placeholder' || l.role === 'textbox') && l.visible && l.matchCount === 1,
|
|
567
|
-
);
|
|
568
|
-
const submitBtnLoc = verifiedLocators.find(
|
|
569
|
-
(l) => l.route === routePathname && l.role === 'button' && !/delete|remove|cancel|close/i.test(l.accessibleName ?? '') && l.visible && l.matchCount >= 1,
|
|
570
|
-
);
|
|
571
|
-
|
|
572
|
-
const resolveLive = (loc: VerifiedLocatorEvidence) => {
|
|
573
|
-
if (loc.testId) return page.getByTestId(loc.testId);
|
|
574
|
-
if (loc.placeholder) return page.getByPlaceholder(loc.placeholder);
|
|
575
|
-
if (loc.label) return page.getByLabel(loc.label);
|
|
576
|
-
if (loc.role && loc.accessibleName) return page.getByRole(loc.role, { name: loc.accessibleName });
|
|
577
|
-
return page.locator(loc.expression);
|
|
578
|
-
};
|
|
579
|
-
|
|
580
|
-
try {
|
|
581
|
-
if (scenario.category === 'persistence') {
|
|
582
|
-
await page.reload({ waitUntil: 'domcontentloaded', timeout: timeoutMs });
|
|
583
|
-
const afterReload = await captureVisibleTextSnapshot(page);
|
|
584
|
-
const preserved = lastCreatedValue ? afterReload.summary.includes(lastCreatedValue) : afterReload.lines.length > 0;
|
|
585
|
-
clientReloadStatus = preserved ? 'PASS' : 'FAIL';
|
|
586
|
-
uiStatus = preserved ? 'PASS' : 'FAIL';
|
|
587
|
-
const actId = `act-${actionRecords.length + 1}`;
|
|
588
|
-
actionRecords.push({
|
|
589
|
-
id: actId,
|
|
590
|
-
scenarioId: scenario.id,
|
|
591
|
-
stepIndex: 0,
|
|
592
|
-
action: 'reload',
|
|
593
|
-
routeBefore: routePathname,
|
|
594
|
-
routeAfter: routePathname,
|
|
595
|
-
domBeforeSummary: beforeSnap.summary,
|
|
596
|
-
domAfterSummary: afterReload.summary,
|
|
597
|
-
domDiff: diffLines(beforeSnap.lines, afterReload.lines),
|
|
598
|
-
networkIds: networkRecords.slice(netStartIdx).map((n) => n.id),
|
|
599
|
-
consoleErrors: consoleErrors.slice(errStartIdx),
|
|
600
|
-
status: 'EXECUTED',
|
|
601
|
-
timestamp: new Date().toISOString(),
|
|
602
|
-
});
|
|
603
|
-
scEvidenceIds.push(actId);
|
|
604
|
-
persistenceObservations.push({
|
|
605
|
-
scenarioId: scenario.id,
|
|
606
|
-
clientReloadPreserved: preserved,
|
|
607
|
-
dbStatus: 'NOT_VERIFIED',
|
|
608
|
-
note: 'page.reload() verifies client/re-fetched state only; does not independently prove backend database persistence.',
|
|
609
|
-
});
|
|
610
|
-
} else if ((scenario.category === 'validation' || scenario.category === 'negative' || scenario.category === 'required-fields') && submitBtnLoc && textInputLoc) {
|
|
611
|
-
const inputHandle = resolveLive(textInputLoc).first();
|
|
612
|
-
const btnHandle = resolveLive(submitBtnLoc).first();
|
|
613
|
-
await inputHandle.fill('');
|
|
614
|
-
await btnHandle.click();
|
|
615
|
-
await page.waitForTimeout(150);
|
|
616
|
-
const afterInvalid = await captureVisibleTextSnapshot(page);
|
|
617
|
-
const diff = diffLines(beforeSnap.lines, afterInvalid.lines);
|
|
618
|
-
const actId = `act-${actionRecords.length + 1}`;
|
|
619
|
-
actionRecords.push({
|
|
620
|
-
id: actId,
|
|
621
|
-
scenarioId: scenario.id,
|
|
622
|
-
stepIndex: 0,
|
|
623
|
-
action: 'submit-empty-validation',
|
|
624
|
-
targetLocatorId: submitBtnLoc.id,
|
|
625
|
-
targetExpression: submitBtnLoc.expression,
|
|
626
|
-
valueRedacted: '',
|
|
627
|
-
routeBefore: routePathname,
|
|
628
|
-
routeAfter: routePathname,
|
|
629
|
-
domBeforeSummary: beforeSnap.summary,
|
|
630
|
-
domAfterSummary: afterInvalid.summary,
|
|
631
|
-
domDiff: diff,
|
|
632
|
-
networkIds: networkRecords.slice(netStartIdx).map((n) => n.id),
|
|
633
|
-
consoleErrors: consoleErrors.slice(errStartIdx),
|
|
634
|
-
status: 'EXECUTED',
|
|
635
|
-
timestamp: new Date().toISOString(),
|
|
636
|
-
});
|
|
637
|
-
scEvidenceIds.push(actId);
|
|
638
|
-
// If no explicit rejection/error failed, record observed behavior or PASS if unchanged / error displayed
|
|
639
|
-
const explicitRejectAc = scenario.expectedOutcomes.some((o) => /\b(reject|prevent|error|remain|not create|ignored)\b/i.test(o));
|
|
640
|
-
const createdSomethingInvalid = diff.some((d) => d.startsWith('+ ') && !/error|invalid|required/i.test(d));
|
|
641
|
-
if (explicitRejectAc && createdSomethingInvalid) {
|
|
642
|
-
uiStatus = 'FAIL';
|
|
643
|
-
findings.push({
|
|
644
|
-
id: `find-${findings.length + 1}`,
|
|
645
|
-
category: 'SPEC_VIOLATION',
|
|
646
|
-
scenarioId: scenario.id,
|
|
647
|
-
evidenceIds: [actId],
|
|
648
|
-
expectedSource: scenario.requirementRefs.join(', '),
|
|
649
|
-
expectedBehavior: scenario.expectedOutcomes.join('; '),
|
|
650
|
-
actualBehavior: `Invalid/empty submission produced unexpected additions: ${diff.slice(0, 3).join('; ')}`,
|
|
651
|
-
confidence: 0.85,
|
|
652
|
-
severity: 'high',
|
|
653
|
-
});
|
|
654
|
-
} else {
|
|
655
|
-
uiStatus = 'PASS';
|
|
656
|
-
findings.push({
|
|
657
|
-
id: `find-${findings.length + 1}`,
|
|
658
|
-
category: 'OBSERVED_BEHAVIOR',
|
|
659
|
-
scenarioId: scenario.id,
|
|
660
|
-
evidenceIds: [actId],
|
|
661
|
-
actualBehavior: diff.length === 0 ? 'Empty submission produced no DOM mutation.' : `Observed DOM diff: ${diff.slice(0, 3).join('; ')}`,
|
|
662
|
-
confidence: 0.85,
|
|
663
|
-
severity: 'info',
|
|
664
|
-
});
|
|
665
|
-
}
|
|
666
|
-
} else if (textInputLoc && allowMutating) {
|
|
667
|
-
const sampleValue = `PIE Verify ${scenario.id.slice(0, 12)}`;
|
|
668
|
-
lastCreatedValue = sampleValue;
|
|
669
|
-
const inputHandle = resolveLive(textInputLoc).first();
|
|
670
|
-
await inputHandle.fill(sampleValue);
|
|
671
|
-
if (submitBtnLoc) {
|
|
672
|
-
await resolveLive(submitBtnLoc).first().click();
|
|
673
|
-
} else {
|
|
674
|
-
await inputHandle.press('Enter');
|
|
675
|
-
}
|
|
676
|
-
await page.waitForTimeout(200);
|
|
677
|
-
const afterSnap = await captureVisibleTextSnapshot(page);
|
|
678
|
-
const diff = diffLines(beforeSnap.lines, afterSnap.lines);
|
|
679
|
-
const actId = `act-${actionRecords.length + 1}`;
|
|
680
|
-
actionRecords.push({
|
|
681
|
-
id: actId,
|
|
682
|
-
scenarioId: scenario.id,
|
|
683
|
-
stepIndex: 0,
|
|
684
|
-
action: submitBtnLoc ? 'fill-and-click' : 'fill-and-enter',
|
|
685
|
-
targetLocatorId: textInputLoc.id,
|
|
686
|
-
targetExpression: textInputLoc.expression,
|
|
687
|
-
valueRedacted: sampleValue,
|
|
688
|
-
routeBefore: routePathname,
|
|
689
|
-
routeAfter: (() => {
|
|
690
|
-
try {
|
|
691
|
-
return new URL(page.url()).pathname || routePathname;
|
|
692
|
-
} catch {
|
|
693
|
-
return routePathname;
|
|
694
|
-
}
|
|
695
|
-
})(),
|
|
696
|
-
domBeforeSummary: beforeSnap.summary,
|
|
697
|
-
domAfterSummary: afterSnap.summary,
|
|
698
|
-
domDiff: diff,
|
|
699
|
-
networkIds: networkRecords.slice(netStartIdx).map((n) => n.id),
|
|
700
|
-
consoleErrors: consoleErrors.slice(errStartIdx),
|
|
701
|
-
status: 'EXECUTED',
|
|
702
|
-
timestamp: new Date().toISOString(),
|
|
703
|
-
});
|
|
704
|
-
scEvidenceIds.push(actId);
|
|
705
|
-
uiStatus = afterSnap.summary.includes(sampleValue) ? 'PASS' : (diff.length > 0 ? 'PASS' : 'FAIL');
|
|
706
|
-
if (uiStatus === 'FAIL' && scenario.provenance === 'explicit-requirement') {
|
|
707
|
-
findings.push({
|
|
708
|
-
id: `find-${findings.length + 1}`,
|
|
709
|
-
category: 'SPEC_VIOLATION',
|
|
710
|
-
scenarioId: scenario.id,
|
|
711
|
-
evidenceIds: [actId],
|
|
712
|
-
expectedSource: scenario.requirementRefs.join(', '),
|
|
713
|
-
expectedBehavior: scenario.expectedOutcomes.join('; '),
|
|
714
|
-
actualBehavior: `Submitted "${sampleValue}" but expected UI change was not observed.`,
|
|
715
|
-
confidence: 0.8,
|
|
716
|
-
severity: 'high',
|
|
717
|
-
});
|
|
718
|
-
}
|
|
719
|
-
} else {
|
|
720
|
-
uiStatus = verifiedLocators.some((l) => l.route === routePathname) ? 'OBSERVED_BEHAVIOR' as any : 'NOT_VERIFIED';
|
|
721
|
-
uiStatus = 'NOT_VERIFIED';
|
|
722
|
-
findings.push({
|
|
723
|
-
id: `find-${findings.length + 1}`,
|
|
724
|
-
category: 'OBSERVED_BEHAVIOR',
|
|
725
|
-
scenarioId: scenario.id,
|
|
726
|
-
evidenceIds: scEvidenceIds,
|
|
727
|
-
actualBehavior: `Inspected route ${routePathname}; discovered ${verifiedLocators.filter((l) => l.route === routePathname).length} visible control(s) without mutating execution.`,
|
|
728
|
-
confidence: 0.7,
|
|
729
|
-
severity: 'info',
|
|
730
|
-
});
|
|
731
|
-
}
|
|
732
|
-
} catch (stepError: any) {
|
|
733
|
-
uiStatus = 'FAIL';
|
|
734
|
-
findings.push({
|
|
735
|
-
id: `find-${findings.length + 1}`,
|
|
736
|
-
category: 'AUTOMATION_GAP',
|
|
737
|
-
scenarioId: scenario.id,
|
|
738
|
-
evidenceIds: scEvidenceIds,
|
|
739
|
-
actualBehavior: `Explorer step error: ${stepError?.message ?? String(stepError)}`,
|
|
740
|
-
confidence: 0.75,
|
|
741
|
-
severity: 'medium',
|
|
742
|
-
});
|
|
743
|
-
}
|
|
744
|
-
|
|
745
|
-
const scenarioNetwork = networkRecords.slice(netStartIdx);
|
|
746
|
-
const scenarioErrors = consoleErrors.slice(errStartIdx);
|
|
747
|
-
const server5xx = scenarioNetwork.filter((n) => n.status >= 500);
|
|
748
|
-
if (server5xx.length > 0 || scenarioErrors.length > 0) {
|
|
749
|
-
findings.push({
|
|
750
|
-
id: `find-${findings.length + 1}`,
|
|
751
|
-
category: 'RUNTIME_ERROR',
|
|
752
|
-
scenarioId: scenario.id,
|
|
753
|
-
evidenceIds: [...scEvidenceIds, ...server5xx.map((n) => n.id)],
|
|
754
|
-
actualBehavior: server5xx.length > 0
|
|
755
|
-
? `HTTP 5xx observed: ${server5xx.map((n) => `${n.method} ${n.url} -> ${n.status}`).join(', ')}`
|
|
756
|
-
: `Runtime console/page error: ${scenarioErrors[0]}`,
|
|
757
|
-
confidence: 0.95,
|
|
758
|
-
severity: 'critical',
|
|
759
|
-
});
|
|
760
|
-
}
|
|
761
|
-
|
|
762
|
-
const apiStatus: VerificationLayerStatus = scenarioNetwork.length === 0
|
|
763
|
-
? 'NOT_APPLICABLE'
|
|
764
|
-
: server5xx.length > 0
|
|
765
|
-
? 'FAIL'
|
|
766
|
-
: scenarioNetwork.some((n) => n.isError)
|
|
767
|
-
? (scenario.category === 'validation' || scenario.category === 'negative' ? 'PASS' : 'FAIL')
|
|
768
|
-
: 'PASS';
|
|
769
|
-
|
|
770
|
-
let dbStatus: VerificationLayerStatus = 'NOT_APPLICABLE';
|
|
771
|
-
if (options.dbVerifier) {
|
|
772
|
-
try {
|
|
773
|
-
const dbRes = await options.dbVerifier(scenario, actionRecords.filter((a) => a.scenarioId === scenario.id));
|
|
774
|
-
dbStatus = dbRes.status;
|
|
775
|
-
} catch {
|
|
776
|
-
dbStatus = 'NOT_VERIFIED';
|
|
777
|
-
}
|
|
778
|
-
}
|
|
779
|
-
|
|
780
|
-
const hasDefect = findings.some(
|
|
781
|
-
(f) => f.scenarioId === scenario.id && (f.category === 'SPEC_VIOLATION' || f.category === 'RUNTIME_ERROR'),
|
|
782
|
-
);
|
|
783
|
-
const finalStatus: ScenarioExplorationStatus['status'] = hasDefect
|
|
784
|
-
? 'DEFECT_DETECTED'
|
|
785
|
-
: uiStatus === 'PASS'
|
|
786
|
-
? 'VERIFIED'
|
|
787
|
-
: 'NOT_VERIFIED';
|
|
788
|
-
|
|
789
|
-
scenarioStatuses.push({
|
|
790
|
-
scenarioId: scenario.id,
|
|
791
|
-
status: finalStatus,
|
|
792
|
-
verificationLayers: {
|
|
793
|
-
UI: uiStatus,
|
|
794
|
-
API: apiStatus,
|
|
795
|
-
CLIENT_RELOAD: clientReloadStatus,
|
|
796
|
-
DB: dbStatus,
|
|
797
|
-
},
|
|
798
|
-
evidenceIds: scEvidenceIds,
|
|
799
|
-
});
|
|
800
|
-
|
|
801
|
-
await persistCheckpoint(true);
|
|
802
|
-
}
|
|
803
|
-
|
|
804
|
-
if (rawNetworkEvents.length > 0) {
|
|
805
|
-
await networkCollector.collect({ events: rawNetworkEvents });
|
|
806
|
-
}
|
|
807
|
-
} catch (error: any) {
|
|
808
|
-
limitations.push(`Browser exploration ended early: ${error?.message ?? String(error)}`);
|
|
809
|
-
await persistCheckpoint(actionRecords.length > 0 || verifiedLocators.length > 0);
|
|
810
|
-
} finally {
|
|
811
|
-
if (browserToClose) {
|
|
812
|
-
await browserToClose.close().catch(() => {});
|
|
813
|
-
}
|
|
814
|
-
}
|
|
815
|
-
|
|
816
|
-
const finalBundle = buildSnapshot(actionRecords.length > 0 || verifiedLocators.length > 0);
|
|
817
|
-
if (options.checkpointPath) {
|
|
818
|
-
await saveExplorationContextBundle(options.checkpointPath, finalBundle);
|
|
819
|
-
}
|
|
820
|
-
return finalBundle;
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
public static async resumeFromCheckpoint(checkpointPath: string): Promise<ExplorationContextBundle> {
|
|
824
|
-
const resolved = path.resolve(checkpointPath);
|
|
825
|
-
await fs.access(resolved);
|
|
826
|
-
return loadExplorationContextBundle(resolved);
|
|
827
|
-
}
|
|
828
|
-
}
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { EvidenceRedactor } from '../evidence/evidence-redactor.js';
|
|
4
|
+
import { NetworkCollector, type RawNetworkEvent } from '../collectors/network-collector.js';
|
|
5
|
+
import { LiveLocatorVerifier } from '../selector/live-locator-verifier.js';
|
|
6
|
+
import type { LocatorPlan } from '../selector/candidate-extractor.js';
|
|
7
|
+
import {
|
|
8
|
+
ExplorationActionRecord,
|
|
9
|
+
ExplorationContextBundle,
|
|
10
|
+
ExplorationFinding,
|
|
11
|
+
ExplorationNetworkRecord,
|
|
12
|
+
loadExplorationContextBundle,
|
|
13
|
+
redactExplorationBundle,
|
|
14
|
+
sanitizeUrlSecrets,
|
|
15
|
+
saveExplorationContextBundle,
|
|
16
|
+
ScenarioExplorationStatus,
|
|
17
|
+
VerificationLayerStatus,
|
|
18
|
+
VerifiedLocatorEvidence,
|
|
19
|
+
} from '../intelligence/evidence-bundle.js';
|
|
20
|
+
|
|
21
|
+
function buildExplorationContextBundleUrlSanitized(baseUrl?: string): string | undefined {
|
|
22
|
+
if (!baseUrl) return undefined;
|
|
23
|
+
return sanitizeUrlSecrets(baseUrl);
|
|
24
|
+
}
|
|
25
|
+
import {
|
|
26
|
+
buildBrownfieldRepositoryGraph,
|
|
27
|
+
type BrownfieldRepositoryGraph,
|
|
28
|
+
} from '../intelligence/repository-profile.js';
|
|
29
|
+
import type {
|
|
30
|
+
GenerationWorkflowPlan,
|
|
31
|
+
PlannedWorkflowScenario,
|
|
32
|
+
} from '../intelligence/workflow-planner.js';
|
|
33
|
+
|
|
34
|
+
export interface ControlledLoginConfig {
|
|
35
|
+
loginUrl: string;
|
|
36
|
+
usernameSelector?: string | undefined;
|
|
37
|
+
passwordSelector?: string | undefined;
|
|
38
|
+
submitSelector?: string | undefined;
|
|
39
|
+
username?: string | undefined;
|
|
40
|
+
password?: string | undefined;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export interface DbVerificationResult {
|
|
44
|
+
status: VerificationLayerStatus;
|
|
45
|
+
summary: string;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
export interface InteractiveExplorerOptions {
|
|
49
|
+
baseUrl?: string | undefined;
|
|
50
|
+
workspaceRoot?: string | undefined;
|
|
51
|
+
workflowPlan: GenerationWorkflowPlan;
|
|
52
|
+
checkpointPath?: string | undefined;
|
|
53
|
+
storageStatePath?: string | undefined;
|
|
54
|
+
loginConfig?: ControlledLoginConfig | undefined;
|
|
55
|
+
allowMutatingActions?: boolean | undefined;
|
|
56
|
+
allowDestructiveActions?: boolean | undefined;
|
|
57
|
+
designatedTestEnvironment?: boolean | undefined;
|
|
58
|
+
maxActionsPerScenario?: number | undefined;
|
|
59
|
+
timeoutMs?: number | undefined;
|
|
60
|
+
browser?: any;
|
|
61
|
+
page?: any;
|
|
62
|
+
dbVerifier?: ((scenario: PlannedWorkflowScenario, actions: ExplorationActionRecord[]) => Promise<DbVerificationResult>) | undefined;
|
|
63
|
+
repositoryGraph?: BrownfieldRepositoryGraph | undefined;
|
|
64
|
+
onCheckpoint?: ((bundle: ExplorationContextBundle) => void | Promise<void>) | undefined;
|
|
65
|
+
simulateCrashAfterScenarioIndex?: number | undefined;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const DESTRUCTIVE_TEXT_RE = /\b(delete\s+account|destroy\s+database|drop\s+table|charge\s+card|complete\s+real\s+payment|wipe\s+all)\b/i;
|
|
69
|
+
|
|
70
|
+
function isLocalhostUrl(url?: string): boolean {
|
|
71
|
+
if (!url) return false;
|
|
72
|
+
return /https?:\/\/(?:localhost|127\.0\.0\.1|\[::1\])(?::\d+)?/i.test(url);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
interface RawDomElementInfo {
|
|
76
|
+
tag: string;
|
|
77
|
+
role: string;
|
|
78
|
+
accessibleName: string;
|
|
79
|
+
placeholder?: string | undefined;
|
|
80
|
+
label?: string | undefined;
|
|
81
|
+
testId?: string | undefined;
|
|
82
|
+
type?: string | undefined;
|
|
83
|
+
visible: boolean;
|
|
84
|
+
frameSelector?: string | undefined;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
function buildLocatorPlanFromDom(el: RawDomElementInfo): {
|
|
88
|
+
strategy: VerifiedLocatorEvidence['strategy'];
|
|
89
|
+
expression: string;
|
|
90
|
+
plan: LocatorPlan;
|
|
91
|
+
} {
|
|
92
|
+
if (el.testId) {
|
|
93
|
+
return {
|
|
94
|
+
strategy: 'testId',
|
|
95
|
+
expression: `page.getByTestId(${JSON.stringify(el.testId)})`,
|
|
96
|
+
plan: { kind: 'getByTestId', testId: el.testId },
|
|
97
|
+
};
|
|
98
|
+
}
|
|
99
|
+
if (el.placeholder) {
|
|
100
|
+
return {
|
|
101
|
+
strategy: 'placeholder',
|
|
102
|
+
expression: `page.getByPlaceholder(${JSON.stringify(el.placeholder)})`,
|
|
103
|
+
plan: { kind: 'getByPlaceholder', placeholder: el.placeholder },
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
if (el.label && (el.tag === 'input' || el.tag === 'textarea' || el.tag === 'select')) {
|
|
107
|
+
return {
|
|
108
|
+
strategy: 'label',
|
|
109
|
+
expression: `page.getByLabel(${JSON.stringify(el.label)})`,
|
|
110
|
+
plan: { kind: 'getByLabel', label: el.label },
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
if (el.role && el.accessibleName) {
|
|
114
|
+
return {
|
|
115
|
+
strategy: 'role',
|
|
116
|
+
expression: `page.getByRole(${JSON.stringify(el.role)}, { name: ${JSON.stringify(el.accessibleName)} })`,
|
|
117
|
+
plan: { kind: 'getByRole', role: el.role, options: { name: el.accessibleName } },
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
return {
|
|
121
|
+
strategy: 'css',
|
|
122
|
+
expression: `page.locator(${JSON.stringify(el.tag)})`,
|
|
123
|
+
plan: { kind: 'locator', selector: el.tag },
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async function captureVisibleTextSnapshot(page: any): Promise<{ summary: string; lines: string[] }> {
|
|
128
|
+
try {
|
|
129
|
+
const raw = await page.evaluate(() => {
|
|
130
|
+
const doc = (globalThis as any).document;
|
|
131
|
+
const bodyText = (doc?.body?.innerText ?? '').trim();
|
|
132
|
+
const lines = bodyText.split(/\r?\n/).map((l: string) => l.trim()).filter(Boolean).slice(0, 60);
|
|
133
|
+
return {
|
|
134
|
+
summary: lines.join(' | ').slice(0, 1200),
|
|
135
|
+
lines,
|
|
136
|
+
};
|
|
137
|
+
});
|
|
138
|
+
return raw;
|
|
139
|
+
} catch {
|
|
140
|
+
return { summary: '', lines: [] };
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
async function collectInteractiveControls(page: any): Promise<RawDomElementInfo[]> {
|
|
145
|
+
try {
|
|
146
|
+
const raw = await page.$$eval(
|
|
147
|
+
'input, textarea, select, button, a[href], [role="button"], [role="textbox"], [role="checkbox"], [role="status"], [role="alert"], h1, h2, h3',
|
|
148
|
+
(nodes: any[]) =>
|
|
149
|
+
nodes.slice(0, 80).map((el) => {
|
|
150
|
+
const tag = el.tagName.toLowerCase();
|
|
151
|
+
const type = el.getAttribute('type') || undefined;
|
|
152
|
+
let role = el.getAttribute('role') || '';
|
|
153
|
+
if (!role) {
|
|
154
|
+
if (tag === 'button' || (tag === 'input' && (type === 'submit' || type === 'button'))) role = 'button';
|
|
155
|
+
else if (tag === 'input' && type === 'checkbox') role = 'checkbox';
|
|
156
|
+
else if (tag === 'input' || tag === 'textarea') role = 'textbox';
|
|
157
|
+
else if (tag === 'select') role = 'combobox';
|
|
158
|
+
else if (tag === 'a') role = 'link';
|
|
159
|
+
else if (/^h[1-3]$/.test(tag)) role = 'heading';
|
|
160
|
+
}
|
|
161
|
+
const placeholder = el.getAttribute('placeholder') || undefined;
|
|
162
|
+
const label = el.getAttribute('aria-label') || el.getAttribute('name') || undefined;
|
|
163
|
+
const testId = el.getAttribute('data-testid') || el.getAttribute('data-test') || undefined;
|
|
164
|
+
const accessibleName = (el.getAttribute('aria-label') || el.innerText || placeholder || '').trim().slice(0, 120);
|
|
165
|
+
const visible = Boolean(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
|
|
166
|
+
return { tag, role, accessibleName, placeholder, label, testId, type, visible };
|
|
167
|
+
}),
|
|
168
|
+
);
|
|
169
|
+
return raw.filter((r: RawDomElementInfo) => r.visible);
|
|
170
|
+
} catch {
|
|
171
|
+
return [];
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
function diffLines(before: string[], after: string[]): string[] {
|
|
176
|
+
const beforeSet = new Set(before);
|
|
177
|
+
const afterSet = new Set(after);
|
|
178
|
+
const diffs: string[] = [];
|
|
179
|
+
for (const line of after) {
|
|
180
|
+
if (!beforeSet.has(line)) diffs.push(`+ ${line}`);
|
|
181
|
+
}
|
|
182
|
+
for (const line of before) {
|
|
183
|
+
if (!afterSet.has(line)) diffs.push(`- ${line}`);
|
|
184
|
+
}
|
|
185
|
+
return diffs.slice(0, 40);
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export class InteractiveWorkflowRunner {
|
|
189
|
+
/**
|
|
190
|
+
* Executes workflow exploration & manual verification in a single native Playwright session
|
|
191
|
+
* with action-level checkpoints, passive defect classification, and redaction.
|
|
192
|
+
*/
|
|
193
|
+
public static async run(options: InteractiveExplorerOptions): Promise<ExplorationContextBundle> {
|
|
194
|
+
const repoGraph = options.repositoryGraph ?? (
|
|
195
|
+
options.workspaceRoot ? await buildBrownfieldRepositoryGraph(options.workspaceRoot) : undefined
|
|
196
|
+
);
|
|
197
|
+
const topology: ExplorationContextBundle['topology'] = !options.baseUrl
|
|
198
|
+
? 'source-only'
|
|
199
|
+
: repoGraph?.topology === 'monorepo'
|
|
200
|
+
? 'monorepo'
|
|
201
|
+
: 'standalone';
|
|
202
|
+
|
|
203
|
+
const bundleId = `explore-${Date.now()}`;
|
|
204
|
+
const networkCollector = new NetworkCollector();
|
|
205
|
+
const rawNetworkEvents: RawNetworkEvent[] = [];
|
|
206
|
+
const networkRecords: ExplorationNetworkRecord[] = [];
|
|
207
|
+
const consoleErrors: string[] = [];
|
|
208
|
+
const verifiedLocators: VerifiedLocatorEvidence[] = [];
|
|
209
|
+
const seenLocatorExpr = new Set<string>();
|
|
210
|
+
const actionRecords: ExplorationActionRecord[] = [];
|
|
211
|
+
const findings: ExplorationFinding[] = [];
|
|
212
|
+
const scenarioStatuses: ScenarioExplorationStatus[] = [];
|
|
213
|
+
const persistenceObservations: ExplorationContextBundle['persistenceObservations'] = [];
|
|
214
|
+
const routesMap = new Map<string, { route: string; title?: string; discoveredAt: string; locatorIds: string[] }>();
|
|
215
|
+
const limitations: string[] = [];
|
|
216
|
+
const evidenceIndex: ExplorationContextBundle['evidenceIndex'] = [];
|
|
217
|
+
|
|
218
|
+
const authMethod: ExplorationContextBundle['authentication']['method'] = options.storageStatePath
|
|
219
|
+
? 'storage-state'
|
|
220
|
+
: options.loginConfig
|
|
221
|
+
? 'login-flow'
|
|
222
|
+
: (repoGraph?.storageStatePaths.length ?? 0) > 0
|
|
223
|
+
? 'existing-fixture'
|
|
224
|
+
: 'none';
|
|
225
|
+
let authenticated = authMethod === 'storage-state';
|
|
226
|
+
|
|
227
|
+
const buildSnapshot = (runtimeVerified: boolean): ExplorationContextBundle => {
|
|
228
|
+
const sanitizedBase = buildExplorationContextBundleUrlSanitized(options.baseUrl);
|
|
229
|
+
const bundle: ExplorationContextBundle = {
|
|
230
|
+
bundleVersion: '1.0',
|
|
231
|
+
bundleId,
|
|
232
|
+
createdAt: new Date().toISOString(),
|
|
233
|
+
topology,
|
|
234
|
+
runtimeVerified,
|
|
235
|
+
...(sanitizedBase ? { baseUrl: sanitizedBase } : {}),
|
|
236
|
+
authentication: {
|
|
237
|
+
configured: authMethod !== 'none',
|
|
238
|
+
method: authMethod,
|
|
239
|
+
authenticated,
|
|
240
|
+
secretsExposed: false,
|
|
241
|
+
},
|
|
242
|
+
requirements: {
|
|
243
|
+
featureName: options.workflowPlan.featureName,
|
|
244
|
+
requirementIds: options.workflowPlan.requirements.map((r) => r.id),
|
|
245
|
+
scenarioIds: options.workflowPlan.scenarios.map((s) => s.id),
|
|
246
|
+
workflowPlanHash: options.workflowPlan.planHash,
|
|
247
|
+
},
|
|
248
|
+
routes: [...routesMap.values()],
|
|
249
|
+
locators: verifiedLocators,
|
|
250
|
+
actions: actionRecords,
|
|
251
|
+
network: networkRecords,
|
|
252
|
+
consoleErrors: [...new Set(consoleErrors)],
|
|
253
|
+
persistenceObservations,
|
|
254
|
+
scenarioStatuses,
|
|
255
|
+
findings,
|
|
256
|
+
...(repoGraph ? { repositoryGraph: repoGraph } : {}),
|
|
257
|
+
evidenceIndex,
|
|
258
|
+
limitations: [...new Set(limitations)],
|
|
259
|
+
redactionComplete: true,
|
|
260
|
+
};
|
|
261
|
+
return redactExplorationBundle(bundle);
|
|
262
|
+
};
|
|
263
|
+
|
|
264
|
+
const persistCheckpoint = async (runtimeVerified: boolean) => {
|
|
265
|
+
const snapshot = buildSnapshot(runtimeVerified);
|
|
266
|
+
if (options.checkpointPath) {
|
|
267
|
+
await saveExplorationContextBundle(options.checkpointPath, snapshot);
|
|
268
|
+
}
|
|
269
|
+
if (options.onCheckpoint) {
|
|
270
|
+
await options.onCheckpoint(snapshot);
|
|
271
|
+
}
|
|
272
|
+
};
|
|
273
|
+
|
|
274
|
+
// Source-only offline mode when no baseUrl is provided
|
|
275
|
+
if (!options.baseUrl) {
|
|
276
|
+
limitations.push('No baseUrl supplied; exploration ran in NOT_RUNTIME_VERIFIED source-only mode.');
|
|
277
|
+
for (const sc of options.workflowPlan.scenarios) {
|
|
278
|
+
scenarioStatuses.push({
|
|
279
|
+
scenarioId: sc.id,
|
|
280
|
+
status: 'NOT_VERIFIED',
|
|
281
|
+
verificationLayers: {
|
|
282
|
+
UI: 'NOT_VERIFIED',
|
|
283
|
+
API: 'NOT_VERIFIED',
|
|
284
|
+
CLIENT_RELOAD: 'NOT_VERIFIED',
|
|
285
|
+
DB: 'NOT_APPLICABLE',
|
|
286
|
+
},
|
|
287
|
+
reason: 'No running application baseUrl supplied.',
|
|
288
|
+
evidenceIds: [],
|
|
289
|
+
});
|
|
290
|
+
}
|
|
291
|
+
const bundle = buildSnapshot(false);
|
|
292
|
+
await persistCheckpoint(false);
|
|
293
|
+
return bundle;
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
let browserToClose: any = null;
|
|
297
|
+
let page = options.page;
|
|
298
|
+
|
|
299
|
+
try {
|
|
300
|
+
if (!page) {
|
|
301
|
+
if (options.browser) {
|
|
302
|
+
const context = await options.browser.newContext({
|
|
303
|
+
...(options.storageStatePath ? { storageState: options.storageStatePath } : {}),
|
|
304
|
+
});
|
|
305
|
+
page = await context.newPage();
|
|
306
|
+
} else {
|
|
307
|
+
const { chromium } = await import('@playwright/test');
|
|
308
|
+
browserToClose = await chromium.launch({ headless: true });
|
|
309
|
+
const context = await browserToClose.newContext({
|
|
310
|
+
...(options.storageStatePath ? { storageState: options.storageStatePath } : {}),
|
|
311
|
+
});
|
|
312
|
+
page = await context.newPage();
|
|
313
|
+
}
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
const timeoutMs = options.timeoutMs ?? 15_000;
|
|
317
|
+
let currentScenarioId: string | undefined;
|
|
318
|
+
|
|
319
|
+
if (typeof page.on === 'function') {
|
|
320
|
+
page.on('response', (res: any) => {
|
|
321
|
+
try {
|
|
322
|
+
const req = typeof res.request === 'function' ? res.request() : undefined;
|
|
323
|
+
const evt: RawNetworkEvent = {
|
|
324
|
+
url: typeof res.url === 'function' ? res.url() : '',
|
|
325
|
+
method: req && typeof req.method === 'function' ? req.method() : 'GET',
|
|
326
|
+
status: typeof res.status === 'function' ? res.status() : 200,
|
|
327
|
+
resourceType: req && typeof req.resourceType === 'function' ? req.resourceType() : undefined,
|
|
328
|
+
timestamp: Date.now(),
|
|
329
|
+
};
|
|
330
|
+
rawNetworkEvents.push(evt);
|
|
331
|
+
const recId = `net-${networkRecords.length + 1}`;
|
|
332
|
+
const isError = evt.status >= 400;
|
|
333
|
+
networkRecords.push({
|
|
334
|
+
id: recId,
|
|
335
|
+
...(currentScenarioId ? { scenarioId: currentScenarioId } : {}),
|
|
336
|
+
method: evt.method,
|
|
337
|
+
url: EvidenceRedactor.redactText(evt.url).redactedText,
|
|
338
|
+
status: evt.status,
|
|
339
|
+
isError,
|
|
340
|
+
...(evt.resourceType ? { resourceType: evt.resourceType } : {}),
|
|
341
|
+
timestamp: new Date(evt.timestamp!).toISOString(),
|
|
342
|
+
});
|
|
343
|
+
} catch {}
|
|
344
|
+
});
|
|
345
|
+
page.on('pageerror', (err: any) => {
|
|
346
|
+
consoleErrors.push(EvidenceRedactor.redactText(err?.message ?? String(err)).redactedText);
|
|
347
|
+
});
|
|
348
|
+
page.on('console', (msg: any) => {
|
|
349
|
+
if (typeof msg.type === 'function' && msg.type() === 'error') {
|
|
350
|
+
consoleErrors.push(EvidenceRedactor.redactText(typeof msg.text === 'function' ? msg.text() : String(msg)).redactedText);
|
|
351
|
+
}
|
|
352
|
+
});
|
|
353
|
+
}
|
|
354
|
+
|
|
355
|
+
// Optional controlled login (credentials NEVER placed in evidence)
|
|
356
|
+
if (options.loginConfig) {
|
|
357
|
+
await page.goto(options.loginConfig.loginUrl, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
|
|
358
|
+
if (options.loginConfig.usernameSelector && options.loginConfig.username) {
|
|
359
|
+
await page.locator(options.loginConfig.usernameSelector).fill(options.loginConfig.username);
|
|
360
|
+
}
|
|
361
|
+
if (options.loginConfig.passwordSelector && options.loginConfig.password) {
|
|
362
|
+
await page.locator(options.loginConfig.passwordSelector).fill(options.loginConfig.password);
|
|
363
|
+
}
|
|
364
|
+
if (options.loginConfig.submitSelector) {
|
|
365
|
+
await page.locator(options.loginConfig.submitSelector).click();
|
|
366
|
+
}
|
|
367
|
+
authenticated = true;
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
const allowDestructive = Boolean(
|
|
371
|
+
options.allowDestructiveActions && (options.designatedTestEnvironment || isLocalhostUrl(options.baseUrl)),
|
|
372
|
+
);
|
|
373
|
+
const allowMutating = options.allowMutatingActions ?? isLocalhostUrl(options.baseUrl);
|
|
374
|
+
let lastCreatedValue: string | undefined;
|
|
375
|
+
|
|
376
|
+
// Multi-route discovery sweep: inspect same-origin links & static routes once before scenarios
|
|
377
|
+
try {
|
|
378
|
+
await page.goto(options.baseUrl, { waitUntil: 'networkidle', timeout: Math.min(timeoutMs, 6_000) }).catch(() =>
|
|
379
|
+
page.goto(options.baseUrl, { waitUntil: 'domcontentloaded', timeout: timeoutMs }),
|
|
380
|
+
);
|
|
381
|
+
const origin = new URL(options.baseUrl).origin;
|
|
382
|
+
const linkHrefs: string[] = await page.$$eval('a[href]', (anchors: any[]) =>
|
|
383
|
+
anchors.map((a) => String(a.getAttribute('href') || '')).filter(Boolean),
|
|
384
|
+
).catch(() => []);
|
|
385
|
+
const candidateRoutes = new Set<string>([options.baseUrl]);
|
|
386
|
+
for (const href of linkHrefs) {
|
|
387
|
+
if (href.startsWith('#') || /^(?:mailto:|tel:|javascript:)/i.test(href)) continue;
|
|
388
|
+
if (/logout|signout|delete|destroy|remove/i.test(href)) continue;
|
|
389
|
+
try {
|
|
390
|
+
const resolved = new URL(href, options.baseUrl);
|
|
391
|
+
if (resolved.origin === origin) candidateRoutes.add(resolved.href);
|
|
392
|
+
} catch {}
|
|
393
|
+
}
|
|
394
|
+
for (const hintRoute of repoGraph?.sourceHints.routes ?? []) {
|
|
395
|
+
if (!hintRoute.startsWith('/') || hintRoute.includes(':') || /logout|signout|delete/i.test(hintRoute)) continue;
|
|
396
|
+
try {
|
|
397
|
+
candidateRoutes.add(new URL(hintRoute, origin).href);
|
|
398
|
+
} catch {}
|
|
399
|
+
}
|
|
400
|
+
const routesToSweep = [...candidateRoutes].slice(0, 5);
|
|
401
|
+
for (const routeUrl of routesToSweep) {
|
|
402
|
+
await page.goto(routeUrl, { waitUntil: 'networkidle', timeout: Math.min(timeoutMs, 4_000) }).catch(() =>
|
|
403
|
+
page.goto(routeUrl, { waitUntil: 'domcontentloaded', timeout: timeoutMs }),
|
|
404
|
+
);
|
|
405
|
+
const sweepPathname = (() => {
|
|
406
|
+
try {
|
|
407
|
+
return new URL(page.url()).pathname || '/';
|
|
408
|
+
} catch {
|
|
409
|
+
return '/';
|
|
410
|
+
}
|
|
411
|
+
})();
|
|
412
|
+
const sweepTitle = typeof page.title === 'function' ? await page.title().catch(() => undefined) : undefined;
|
|
413
|
+
if (!routesMap.has(sweepPathname)) {
|
|
414
|
+
routesMap.set(sweepPathname, {
|
|
415
|
+
route: sweepPathname,
|
|
416
|
+
...(sweepTitle ? { title: sweepTitle } : {}),
|
|
417
|
+
discoveredAt: new Date().toISOString(),
|
|
418
|
+
locatorIds: [],
|
|
419
|
+
});
|
|
420
|
+
}
|
|
421
|
+
const sweepControls = await collectInteractiveControls(page);
|
|
422
|
+
for (const ctrl of sweepControls) {
|
|
423
|
+
const built = buildLocatorPlanFromDom(ctrl);
|
|
424
|
+
if (seenLocatorExpr.has(`${sweepPathname}::${built.expression}`)) continue;
|
|
425
|
+
const ver = await LiveLocatorVerifier.verify(page, built.plan);
|
|
426
|
+
const locId = `loc-${verifiedLocators.length + 1}`;
|
|
427
|
+
seenLocatorExpr.add(`${sweepPathname}::${built.expression}`);
|
|
428
|
+
verifiedLocators.push({
|
|
429
|
+
id: locId,
|
|
430
|
+
route: sweepPathname,
|
|
431
|
+
strategy: built.strategy,
|
|
432
|
+
expression: built.expression,
|
|
433
|
+
...(ctrl.role ? { role: ctrl.role } : {}),
|
|
434
|
+
...(ctrl.accessibleName ? { accessibleName: ctrl.accessibleName } : {}),
|
|
435
|
+
...(ctrl.label ? { label: ctrl.label } : {}),
|
|
436
|
+
...(ctrl.placeholder ? { placeholder: ctrl.placeholder } : {}),
|
|
437
|
+
...(ctrl.testId ? { testId: ctrl.testId } : {}),
|
|
438
|
+
matchCount: ver.count,
|
|
439
|
+
visible: ver.isVisible,
|
|
440
|
+
enabled: ver.isEnabled,
|
|
441
|
+
confidence: ver.isUnique ? 0.95 : 0.5,
|
|
442
|
+
timestamp: new Date().toISOString(),
|
|
443
|
+
});
|
|
444
|
+
routesMap.get(sweepPathname)!.locatorIds.push(locId);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
} catch {
|
|
448
|
+
// Ignore non-fatal route sweep errors; scenario loop continues directly on baseUrl
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
for (let scIdx = 0; scIdx < options.workflowPlan.scenarios.length; scIdx += 1) {
|
|
452
|
+
const scenario = options.workflowPlan.scenarios[scIdx]!;
|
|
453
|
+
currentScenarioId = scenario.id;
|
|
454
|
+
|
|
455
|
+
if (options.simulateCrashAfterScenarioIndex !== undefined && scIdx >= options.simulateCrashAfterScenarioIndex) {
|
|
456
|
+
throw new Error(`Simulated exploration interruption before scenario '${scenario.id}'`);
|
|
457
|
+
}
|
|
458
|
+
|
|
459
|
+
if (scenario.applicability === 'needs_clarification') {
|
|
460
|
+
scenarioStatuses.push({
|
|
461
|
+
scenarioId: scenario.id,
|
|
462
|
+
status: 'NOT_VERIFIED',
|
|
463
|
+
verificationLayers: { UI: 'NOT_VERIFIED', API: 'NOT_APPLICABLE', CLIENT_RELOAD: 'NOT_APPLICABLE', DB: 'NOT_APPLICABLE' },
|
|
464
|
+
reason: 'Scenario marked needs_clarification in workflow plan.',
|
|
465
|
+
evidenceIds: [],
|
|
466
|
+
});
|
|
467
|
+
continue;
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
const netStartIdx = networkRecords.length;
|
|
471
|
+
const errStartIdx = consoleErrors.length;
|
|
472
|
+
const scEvidenceIds: string[] = [];
|
|
473
|
+
|
|
474
|
+
// Navigate to baseUrl for deterministic isolation (skip reload scenarios so state stays)
|
|
475
|
+
if (scenario.category !== 'persistence') {
|
|
476
|
+
await page.goto(options.baseUrl, { waitUntil: 'domcontentloaded', timeout: timeoutMs });
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
const routePathname = (() => {
|
|
480
|
+
try {
|
|
481
|
+
return new URL(page.url()).pathname || '/';
|
|
482
|
+
} catch {
|
|
483
|
+
return '/';
|
|
484
|
+
}
|
|
485
|
+
})();
|
|
486
|
+
const pageTitle = typeof page.title === 'function' ? await page.title().catch(() => undefined) : undefined;
|
|
487
|
+
if (!routesMap.has(routePathname)) {
|
|
488
|
+
routesMap.set(routePathname, {
|
|
489
|
+
route: routePathname,
|
|
490
|
+
...(pageTitle ? { title: pageTitle } : {}),
|
|
491
|
+
discoveredAt: new Date().toISOString(),
|
|
492
|
+
locatorIds: [],
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
const controls = await collectInteractiveControls(page);
|
|
497
|
+
for (const ctrl of controls) {
|
|
498
|
+
const built = buildLocatorPlanFromDom(ctrl);
|
|
499
|
+
if (seenLocatorExpr.has(`${routePathname}::${built.expression}`)) continue;
|
|
500
|
+
const ver = await LiveLocatorVerifier.verify(page, built.plan);
|
|
501
|
+
const locId = `loc-${verifiedLocators.length + 1}`;
|
|
502
|
+
seenLocatorExpr.add(`${routePathname}::${built.expression}`);
|
|
503
|
+
verifiedLocators.push({
|
|
504
|
+
id: locId,
|
|
505
|
+
route: routePathname,
|
|
506
|
+
strategy: built.strategy,
|
|
507
|
+
expression: built.expression,
|
|
508
|
+
...(ctrl.role ? { role: ctrl.role } : {}),
|
|
509
|
+
...(ctrl.accessibleName ? { accessibleName: ctrl.accessibleName } : {}),
|
|
510
|
+
...(ctrl.label ? { label: ctrl.label } : {}),
|
|
511
|
+
...(ctrl.placeholder ? { placeholder: ctrl.placeholder } : {}),
|
|
512
|
+
...(ctrl.testId ? { testId: ctrl.testId } : {}),
|
|
513
|
+
matchCount: ver.count,
|
|
514
|
+
visible: ver.isVisible,
|
|
515
|
+
enabled: ver.isEnabled,
|
|
516
|
+
confidence: ver.isUnique ? 0.95 : 0.5,
|
|
517
|
+
timestamp: new Date().toISOString(),
|
|
518
|
+
});
|
|
519
|
+
routesMap.get(routePathname)!.locatorIds.push(locId);
|
|
520
|
+
scEvidenceIds.push(locId);
|
|
521
|
+
evidenceIndex.push({
|
|
522
|
+
id: locId,
|
|
523
|
+
kind: 'verified-locator',
|
|
524
|
+
source: 'playwright-live-dom',
|
|
525
|
+
timestamp: new Date().toISOString(),
|
|
526
|
+
confidence: ver.isUnique ? 0.95 : 0.5,
|
|
527
|
+
scenarioIds: [scenario.id],
|
|
528
|
+
description: `${built.expression} (count=${ver.count})`,
|
|
529
|
+
});
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
// Check destructive action guard against scenario text
|
|
533
|
+
const combinedScenarioText = `${scenario.title} ${scenario.steps.map((s) => s.text).join(' ')}`;
|
|
534
|
+
if (DESTRUCTIVE_TEXT_RE.test(combinedScenarioText) && !allowDestructive) {
|
|
535
|
+
actionRecords.push({
|
|
536
|
+
id: `act-${actionRecords.length + 1}`,
|
|
537
|
+
scenarioId: scenario.id,
|
|
538
|
+
stepIndex: 0,
|
|
539
|
+
action: 'destructive-policy-check',
|
|
540
|
+
routeBefore: routePathname,
|
|
541
|
+
routeAfter: routePathname,
|
|
542
|
+
domBeforeSummary: '',
|
|
543
|
+
domAfterSummary: '',
|
|
544
|
+
domDiff: [],
|
|
545
|
+
networkIds: [],
|
|
546
|
+
consoleErrors: [],
|
|
547
|
+
status: 'BLOCKED_BY_POLICY',
|
|
548
|
+
timestamp: new Date().toISOString(),
|
|
549
|
+
});
|
|
550
|
+
scenarioStatuses.push({
|
|
551
|
+
scenarioId: scenario.id,
|
|
552
|
+
status: 'BLOCKED',
|
|
553
|
+
verificationLayers: { UI: 'NOT_VERIFIED', API: 'NOT_VERIFIED', CLIENT_RELOAD: 'NOT_APPLICABLE', DB: 'NOT_APPLICABLE' },
|
|
554
|
+
reason: 'Destructive operation blocked by safety policy (requires designatedTestEnvironment & allowDestructiveActions).',
|
|
555
|
+
evidenceIds: scEvidenceIds,
|
|
556
|
+
});
|
|
557
|
+
await persistCheckpoint(true);
|
|
558
|
+
continue;
|
|
559
|
+
}
|
|
560
|
+
|
|
561
|
+
const beforeSnap = await captureVisibleTextSnapshot(page);
|
|
562
|
+
let uiStatus: VerificationLayerStatus = 'NOT_VERIFIED';
|
|
563
|
+
let clientReloadStatus: VerificationLayerStatus = 'NOT_APPLICABLE';
|
|
564
|
+
|
|
565
|
+
const textInputLoc = verifiedLocators.find(
|
|
566
|
+
(l) => l.route === routePathname && (l.strategy === 'placeholder' || l.role === 'textbox') && l.visible && l.matchCount === 1,
|
|
567
|
+
);
|
|
568
|
+
const submitBtnLoc = verifiedLocators.find(
|
|
569
|
+
(l) => l.route === routePathname && l.role === 'button' && !/delete|remove|cancel|close/i.test(l.accessibleName ?? '') && l.visible && l.matchCount >= 1,
|
|
570
|
+
);
|
|
571
|
+
|
|
572
|
+
const resolveLive = (loc: VerifiedLocatorEvidence) => {
|
|
573
|
+
if (loc.testId) return page.getByTestId(loc.testId);
|
|
574
|
+
if (loc.placeholder) return page.getByPlaceholder(loc.placeholder);
|
|
575
|
+
if (loc.label) return page.getByLabel(loc.label);
|
|
576
|
+
if (loc.role && loc.accessibleName) return page.getByRole(loc.role, { name: loc.accessibleName });
|
|
577
|
+
return page.locator(loc.expression);
|
|
578
|
+
};
|
|
579
|
+
|
|
580
|
+
try {
|
|
581
|
+
if (scenario.category === 'persistence') {
|
|
582
|
+
await page.reload({ waitUntil: 'domcontentloaded', timeout: timeoutMs });
|
|
583
|
+
const afterReload = await captureVisibleTextSnapshot(page);
|
|
584
|
+
const preserved = lastCreatedValue ? afterReload.summary.includes(lastCreatedValue) : afterReload.lines.length > 0;
|
|
585
|
+
clientReloadStatus = preserved ? 'PASS' : 'FAIL';
|
|
586
|
+
uiStatus = preserved ? 'PASS' : 'FAIL';
|
|
587
|
+
const actId = `act-${actionRecords.length + 1}`;
|
|
588
|
+
actionRecords.push({
|
|
589
|
+
id: actId,
|
|
590
|
+
scenarioId: scenario.id,
|
|
591
|
+
stepIndex: 0,
|
|
592
|
+
action: 'reload',
|
|
593
|
+
routeBefore: routePathname,
|
|
594
|
+
routeAfter: routePathname,
|
|
595
|
+
domBeforeSummary: beforeSnap.summary,
|
|
596
|
+
domAfterSummary: afterReload.summary,
|
|
597
|
+
domDiff: diffLines(beforeSnap.lines, afterReload.lines),
|
|
598
|
+
networkIds: networkRecords.slice(netStartIdx).map((n) => n.id),
|
|
599
|
+
consoleErrors: consoleErrors.slice(errStartIdx),
|
|
600
|
+
status: 'EXECUTED',
|
|
601
|
+
timestamp: new Date().toISOString(),
|
|
602
|
+
});
|
|
603
|
+
scEvidenceIds.push(actId);
|
|
604
|
+
persistenceObservations.push({
|
|
605
|
+
scenarioId: scenario.id,
|
|
606
|
+
clientReloadPreserved: preserved,
|
|
607
|
+
dbStatus: 'NOT_VERIFIED',
|
|
608
|
+
note: 'page.reload() verifies client/re-fetched state only; does not independently prove backend database persistence.',
|
|
609
|
+
});
|
|
610
|
+
} else if ((scenario.category === 'validation' || scenario.category === 'negative' || scenario.category === 'required-fields') && submitBtnLoc && textInputLoc) {
|
|
611
|
+
const inputHandle = resolveLive(textInputLoc).first();
|
|
612
|
+
const btnHandle = resolveLive(submitBtnLoc).first();
|
|
613
|
+
await inputHandle.fill('');
|
|
614
|
+
await btnHandle.click();
|
|
615
|
+
await page.waitForTimeout(150);
|
|
616
|
+
const afterInvalid = await captureVisibleTextSnapshot(page);
|
|
617
|
+
const diff = diffLines(beforeSnap.lines, afterInvalid.lines);
|
|
618
|
+
const actId = `act-${actionRecords.length + 1}`;
|
|
619
|
+
actionRecords.push({
|
|
620
|
+
id: actId,
|
|
621
|
+
scenarioId: scenario.id,
|
|
622
|
+
stepIndex: 0,
|
|
623
|
+
action: 'submit-empty-validation',
|
|
624
|
+
targetLocatorId: submitBtnLoc.id,
|
|
625
|
+
targetExpression: submitBtnLoc.expression,
|
|
626
|
+
valueRedacted: '',
|
|
627
|
+
routeBefore: routePathname,
|
|
628
|
+
routeAfter: routePathname,
|
|
629
|
+
domBeforeSummary: beforeSnap.summary,
|
|
630
|
+
domAfterSummary: afterInvalid.summary,
|
|
631
|
+
domDiff: diff,
|
|
632
|
+
networkIds: networkRecords.slice(netStartIdx).map((n) => n.id),
|
|
633
|
+
consoleErrors: consoleErrors.slice(errStartIdx),
|
|
634
|
+
status: 'EXECUTED',
|
|
635
|
+
timestamp: new Date().toISOString(),
|
|
636
|
+
});
|
|
637
|
+
scEvidenceIds.push(actId);
|
|
638
|
+
// If no explicit rejection/error failed, record observed behavior or PASS if unchanged / error displayed
|
|
639
|
+
const explicitRejectAc = scenario.expectedOutcomes.some((o) => /\b(reject|prevent|error|remain|not create|ignored)\b/i.test(o));
|
|
640
|
+
const createdSomethingInvalid = diff.some((d) => d.startsWith('+ ') && !/error|invalid|required/i.test(d));
|
|
641
|
+
if (explicitRejectAc && createdSomethingInvalid) {
|
|
642
|
+
uiStatus = 'FAIL';
|
|
643
|
+
findings.push({
|
|
644
|
+
id: `find-${findings.length + 1}`,
|
|
645
|
+
category: 'SPEC_VIOLATION',
|
|
646
|
+
scenarioId: scenario.id,
|
|
647
|
+
evidenceIds: [actId],
|
|
648
|
+
expectedSource: scenario.requirementRefs.join(', '),
|
|
649
|
+
expectedBehavior: scenario.expectedOutcomes.join('; '),
|
|
650
|
+
actualBehavior: `Invalid/empty submission produced unexpected additions: ${diff.slice(0, 3).join('; ')}`,
|
|
651
|
+
confidence: 0.85,
|
|
652
|
+
severity: 'high',
|
|
653
|
+
});
|
|
654
|
+
} else {
|
|
655
|
+
uiStatus = 'PASS';
|
|
656
|
+
findings.push({
|
|
657
|
+
id: `find-${findings.length + 1}`,
|
|
658
|
+
category: 'OBSERVED_BEHAVIOR',
|
|
659
|
+
scenarioId: scenario.id,
|
|
660
|
+
evidenceIds: [actId],
|
|
661
|
+
actualBehavior: diff.length === 0 ? 'Empty submission produced no DOM mutation.' : `Observed DOM diff: ${diff.slice(0, 3).join('; ')}`,
|
|
662
|
+
confidence: 0.85,
|
|
663
|
+
severity: 'info',
|
|
664
|
+
});
|
|
665
|
+
}
|
|
666
|
+
} else if (textInputLoc && allowMutating) {
|
|
667
|
+
const sampleValue = `PIE Verify ${scenario.id.slice(0, 12)}`;
|
|
668
|
+
lastCreatedValue = sampleValue;
|
|
669
|
+
const inputHandle = resolveLive(textInputLoc).first();
|
|
670
|
+
await inputHandle.fill(sampleValue);
|
|
671
|
+
if (submitBtnLoc) {
|
|
672
|
+
await resolveLive(submitBtnLoc).first().click();
|
|
673
|
+
} else {
|
|
674
|
+
await inputHandle.press('Enter');
|
|
675
|
+
}
|
|
676
|
+
await page.waitForTimeout(200);
|
|
677
|
+
const afterSnap = await captureVisibleTextSnapshot(page);
|
|
678
|
+
const diff = diffLines(beforeSnap.lines, afterSnap.lines);
|
|
679
|
+
const actId = `act-${actionRecords.length + 1}`;
|
|
680
|
+
actionRecords.push({
|
|
681
|
+
id: actId,
|
|
682
|
+
scenarioId: scenario.id,
|
|
683
|
+
stepIndex: 0,
|
|
684
|
+
action: submitBtnLoc ? 'fill-and-click' : 'fill-and-enter',
|
|
685
|
+
targetLocatorId: textInputLoc.id,
|
|
686
|
+
targetExpression: textInputLoc.expression,
|
|
687
|
+
valueRedacted: sampleValue,
|
|
688
|
+
routeBefore: routePathname,
|
|
689
|
+
routeAfter: (() => {
|
|
690
|
+
try {
|
|
691
|
+
return new URL(page.url()).pathname || routePathname;
|
|
692
|
+
} catch {
|
|
693
|
+
return routePathname;
|
|
694
|
+
}
|
|
695
|
+
})(),
|
|
696
|
+
domBeforeSummary: beforeSnap.summary,
|
|
697
|
+
domAfterSummary: afterSnap.summary,
|
|
698
|
+
domDiff: diff,
|
|
699
|
+
networkIds: networkRecords.slice(netStartIdx).map((n) => n.id),
|
|
700
|
+
consoleErrors: consoleErrors.slice(errStartIdx),
|
|
701
|
+
status: 'EXECUTED',
|
|
702
|
+
timestamp: new Date().toISOString(),
|
|
703
|
+
});
|
|
704
|
+
scEvidenceIds.push(actId);
|
|
705
|
+
uiStatus = afterSnap.summary.includes(sampleValue) ? 'PASS' : (diff.length > 0 ? 'PASS' : 'FAIL');
|
|
706
|
+
if (uiStatus === 'FAIL' && scenario.provenance === 'explicit-requirement') {
|
|
707
|
+
findings.push({
|
|
708
|
+
id: `find-${findings.length + 1}`,
|
|
709
|
+
category: 'SPEC_VIOLATION',
|
|
710
|
+
scenarioId: scenario.id,
|
|
711
|
+
evidenceIds: [actId],
|
|
712
|
+
expectedSource: scenario.requirementRefs.join(', '),
|
|
713
|
+
expectedBehavior: scenario.expectedOutcomes.join('; '),
|
|
714
|
+
actualBehavior: `Submitted "${sampleValue}" but expected UI change was not observed.`,
|
|
715
|
+
confidence: 0.8,
|
|
716
|
+
severity: 'high',
|
|
717
|
+
});
|
|
718
|
+
}
|
|
719
|
+
} else {
|
|
720
|
+
uiStatus = verifiedLocators.some((l) => l.route === routePathname) ? 'OBSERVED_BEHAVIOR' as any : 'NOT_VERIFIED';
|
|
721
|
+
uiStatus = 'NOT_VERIFIED';
|
|
722
|
+
findings.push({
|
|
723
|
+
id: `find-${findings.length + 1}`,
|
|
724
|
+
category: 'OBSERVED_BEHAVIOR',
|
|
725
|
+
scenarioId: scenario.id,
|
|
726
|
+
evidenceIds: scEvidenceIds,
|
|
727
|
+
actualBehavior: `Inspected route ${routePathname}; discovered ${verifiedLocators.filter((l) => l.route === routePathname).length} visible control(s) without mutating execution.`,
|
|
728
|
+
confidence: 0.7,
|
|
729
|
+
severity: 'info',
|
|
730
|
+
});
|
|
731
|
+
}
|
|
732
|
+
} catch (stepError: any) {
|
|
733
|
+
uiStatus = 'FAIL';
|
|
734
|
+
findings.push({
|
|
735
|
+
id: `find-${findings.length + 1}`,
|
|
736
|
+
category: 'AUTOMATION_GAP',
|
|
737
|
+
scenarioId: scenario.id,
|
|
738
|
+
evidenceIds: scEvidenceIds,
|
|
739
|
+
actualBehavior: `Explorer step error: ${stepError?.message ?? String(stepError)}`,
|
|
740
|
+
confidence: 0.75,
|
|
741
|
+
severity: 'medium',
|
|
742
|
+
});
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
const scenarioNetwork = networkRecords.slice(netStartIdx);
|
|
746
|
+
const scenarioErrors = consoleErrors.slice(errStartIdx);
|
|
747
|
+
const server5xx = scenarioNetwork.filter((n) => n.status >= 500);
|
|
748
|
+
if (server5xx.length > 0 || scenarioErrors.length > 0) {
|
|
749
|
+
findings.push({
|
|
750
|
+
id: `find-${findings.length + 1}`,
|
|
751
|
+
category: 'RUNTIME_ERROR',
|
|
752
|
+
scenarioId: scenario.id,
|
|
753
|
+
evidenceIds: [...scEvidenceIds, ...server5xx.map((n) => n.id)],
|
|
754
|
+
actualBehavior: server5xx.length > 0
|
|
755
|
+
? `HTTP 5xx observed: ${server5xx.map((n) => `${n.method} ${n.url} -> ${n.status}`).join(', ')}`
|
|
756
|
+
: `Runtime console/page error: ${scenarioErrors[0]}`,
|
|
757
|
+
confidence: 0.95,
|
|
758
|
+
severity: 'critical',
|
|
759
|
+
});
|
|
760
|
+
}
|
|
761
|
+
|
|
762
|
+
const apiStatus: VerificationLayerStatus = scenarioNetwork.length === 0
|
|
763
|
+
? 'NOT_APPLICABLE'
|
|
764
|
+
: server5xx.length > 0
|
|
765
|
+
? 'FAIL'
|
|
766
|
+
: scenarioNetwork.some((n) => n.isError)
|
|
767
|
+
? (scenario.category === 'validation' || scenario.category === 'negative' ? 'PASS' : 'FAIL')
|
|
768
|
+
: 'PASS';
|
|
769
|
+
|
|
770
|
+
let dbStatus: VerificationLayerStatus = 'NOT_APPLICABLE';
|
|
771
|
+
if (options.dbVerifier) {
|
|
772
|
+
try {
|
|
773
|
+
const dbRes = await options.dbVerifier(scenario, actionRecords.filter((a) => a.scenarioId === scenario.id));
|
|
774
|
+
dbStatus = dbRes.status;
|
|
775
|
+
} catch {
|
|
776
|
+
dbStatus = 'NOT_VERIFIED';
|
|
777
|
+
}
|
|
778
|
+
}
|
|
779
|
+
|
|
780
|
+
const hasDefect = findings.some(
|
|
781
|
+
(f) => f.scenarioId === scenario.id && (f.category === 'SPEC_VIOLATION' || f.category === 'RUNTIME_ERROR'),
|
|
782
|
+
);
|
|
783
|
+
const finalStatus: ScenarioExplorationStatus['status'] = hasDefect
|
|
784
|
+
? 'DEFECT_DETECTED'
|
|
785
|
+
: uiStatus === 'PASS'
|
|
786
|
+
? 'VERIFIED'
|
|
787
|
+
: 'NOT_VERIFIED';
|
|
788
|
+
|
|
789
|
+
scenarioStatuses.push({
|
|
790
|
+
scenarioId: scenario.id,
|
|
791
|
+
status: finalStatus,
|
|
792
|
+
verificationLayers: {
|
|
793
|
+
UI: uiStatus,
|
|
794
|
+
API: apiStatus,
|
|
795
|
+
CLIENT_RELOAD: clientReloadStatus,
|
|
796
|
+
DB: dbStatus,
|
|
797
|
+
},
|
|
798
|
+
evidenceIds: scEvidenceIds,
|
|
799
|
+
});
|
|
800
|
+
|
|
801
|
+
await persistCheckpoint(true);
|
|
802
|
+
}
|
|
803
|
+
|
|
804
|
+
if (rawNetworkEvents.length > 0) {
|
|
805
|
+
await networkCollector.collect({ events: rawNetworkEvents });
|
|
806
|
+
}
|
|
807
|
+
} catch (error: any) {
|
|
808
|
+
limitations.push(`Browser exploration ended early: ${error?.message ?? String(error)}`);
|
|
809
|
+
await persistCheckpoint(actionRecords.length > 0 || verifiedLocators.length > 0);
|
|
810
|
+
} finally {
|
|
811
|
+
if (browserToClose) {
|
|
812
|
+
await browserToClose.close().catch(() => {});
|
|
813
|
+
}
|
|
814
|
+
}
|
|
815
|
+
|
|
816
|
+
const finalBundle = buildSnapshot(actionRecords.length > 0 || verifiedLocators.length > 0);
|
|
817
|
+
if (options.checkpointPath) {
|
|
818
|
+
await saveExplorationContextBundle(options.checkpointPath, finalBundle);
|
|
819
|
+
}
|
|
820
|
+
return finalBundle;
|
|
821
|
+
}
|
|
822
|
+
|
|
823
|
+
public static async resumeFromCheckpoint(checkpointPath: string): Promise<ExplorationContextBundle> {
|
|
824
|
+
const resolved = path.resolve(checkpointPath);
|
|
825
|
+
await fs.access(resolved);
|
|
826
|
+
return loadExplorationContextBundle(resolved);
|
|
827
|
+
}
|
|
828
|
+
}
|