testchimp-runner-core 0.0.34 → 0.0.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/execution-service.d.ts +1 -4
- package/dist/execution-service.d.ts.map +1 -1
- package/dist/execution-service.js +155 -468
- package/dist/execution-service.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +11 -1
- package/dist/index.js.map +1 -1
- package/dist/orchestrator/decision-parser.d.ts +18 -0
- package/dist/orchestrator/decision-parser.d.ts.map +1 -0
- package/dist/orchestrator/decision-parser.js +127 -0
- package/dist/orchestrator/decision-parser.js.map +1 -0
- package/dist/orchestrator/index.d.ts +4 -2
- package/dist/orchestrator/index.d.ts.map +1 -1
- package/dist/orchestrator/index.js +14 -2
- package/dist/orchestrator/index.js.map +1 -1
- package/dist/orchestrator/orchestrator-agent.d.ts +17 -14
- package/dist/orchestrator/orchestrator-agent.d.ts.map +1 -1
- package/dist/orchestrator/orchestrator-agent.js +534 -204
- package/dist/orchestrator/orchestrator-agent.js.map +1 -1
- package/dist/orchestrator/orchestrator-prompts.d.ts +14 -2
- package/dist/orchestrator/orchestrator-prompts.d.ts.map +1 -1
- package/dist/orchestrator/orchestrator-prompts.js +529 -247
- package/dist/orchestrator/orchestrator-prompts.js.map +1 -1
- package/dist/orchestrator/page-som-handler.d.ts +106 -0
- package/dist/orchestrator/page-som-handler.d.ts.map +1 -0
- package/dist/orchestrator/page-som-handler.js +1353 -0
- package/dist/orchestrator/page-som-handler.js.map +1 -0
- package/dist/orchestrator/som-types.d.ts +149 -0
- package/dist/orchestrator/som-types.d.ts.map +1 -0
- package/dist/orchestrator/som-types.js +87 -0
- package/dist/orchestrator/som-types.js.map +1 -0
- package/dist/orchestrator/tool-registry.d.ts +2 -0
- package/dist/orchestrator/tool-registry.d.ts.map +1 -1
- package/dist/orchestrator/tool-registry.js.map +1 -1
- package/dist/orchestrator/tools/index.d.ts +4 -1
- package/dist/orchestrator/tools/index.d.ts.map +1 -1
- package/dist/orchestrator/tools/index.js +7 -2
- package/dist/orchestrator/tools/index.js.map +1 -1
- package/dist/orchestrator/tools/refresh-som-markers.d.ts +12 -0
- package/dist/orchestrator/tools/refresh-som-markers.d.ts.map +1 -0
- package/dist/orchestrator/tools/refresh-som-markers.js +64 -0
- package/dist/orchestrator/tools/refresh-som-markers.js.map +1 -0
- package/dist/orchestrator/tools/view-previous-screenshot.d.ts +15 -0
- package/dist/orchestrator/tools/view-previous-screenshot.d.ts.map +1 -0
- package/dist/orchestrator/tools/view-previous-screenshot.js +92 -0
- package/dist/orchestrator/tools/view-previous-screenshot.js.map +1 -0
- package/dist/orchestrator/types.d.ts +23 -1
- package/dist/orchestrator/types.d.ts.map +1 -1
- package/dist/orchestrator/types.js +11 -1
- package/dist/orchestrator/types.js.map +1 -1
- package/dist/scenario-service.d.ts +5 -0
- package/dist/scenario-service.d.ts.map +1 -1
- package/dist/scenario-service.js +17 -0
- package/dist/scenario-service.js.map +1 -1
- package/dist/scenario-worker-class.d.ts +4 -0
- package/dist/scenario-worker-class.d.ts.map +1 -1
- package/dist/scenario-worker-class.js +18 -3
- package/dist/scenario-worker-class.js.map +1 -1
- package/dist/testing/agent-tester.d.ts +35 -0
- package/dist/testing/agent-tester.d.ts.map +1 -0
- package/dist/testing/agent-tester.js +84 -0
- package/dist/testing/agent-tester.js.map +1 -0
- package/dist/testing/ref-translator-tester.d.ts +44 -0
- package/dist/testing/ref-translator-tester.d.ts.map +1 -0
- package/dist/testing/ref-translator-tester.js +104 -0
- package/dist/testing/ref-translator-tester.js.map +1 -0
- package/dist/utils/hierarchical-selector.d.ts +47 -0
- package/dist/utils/hierarchical-selector.d.ts.map +1 -0
- package/dist/utils/hierarchical-selector.js +212 -0
- package/dist/utils/hierarchical-selector.js.map +1 -0
- package/dist/utils/page-info-retry.d.ts +14 -0
- package/dist/utils/page-info-retry.d.ts.map +1 -0
- package/dist/utils/page-info-retry.js +60 -0
- package/dist/utils/page-info-retry.js.map +1 -0
- package/dist/utils/page-info-utils.d.ts +1 -0
- package/dist/utils/page-info-utils.d.ts.map +1 -1
- package/dist/utils/page-info-utils.js +46 -18
- package/dist/utils/page-info-utils.js.map +1 -1
- package/dist/utils/ref-attacher.d.ts +21 -0
- package/dist/utils/ref-attacher.d.ts.map +1 -0
- package/dist/utils/ref-attacher.js +149 -0
- package/dist/utils/ref-attacher.js.map +1 -0
- package/dist/utils/ref-translator.d.ts +49 -0
- package/dist/utils/ref-translator.d.ts.map +1 -0
- package/dist/utils/ref-translator.js +276 -0
- package/dist/utils/ref-translator.js.map +1 -0
- package/package.json +6 -1
- package/RELEASE_0.0.26.md +0 -165
- package/RELEASE_0.0.27.md +0 -236
- package/RELEASE_0.0.28.md +0 -286
- package/plandocs/BEFORE_AFTER_VERIFICATION.md +0 -148
- package/plandocs/COORDINATE_MODE_DIAGNOSIS.md +0 -144
- package/plandocs/CREDIT_CALLBACK_ARCHITECTURE.md +0 -253
- package/plandocs/HUMAN_LIKE_IMPROVEMENTS.md +0 -642
- package/plandocs/IMPLEMENTATION_STATUS.md +0 -108
- package/plandocs/INTEGRATION_COMPLETE.md +0 -322
- package/plandocs/MULTI_AGENT_ARCHITECTURE_REVIEW.md +0 -844
- package/plandocs/ORCHESTRATOR_MVP_SUMMARY.md +0 -539
- package/plandocs/PHASE1_ABSTRACTION_COMPLETE.md +0 -241
- package/plandocs/PHASE1_FINAL_STATUS.md +0 -210
- package/plandocs/PHASE_1_COMPLETE.md +0 -165
- package/plandocs/PHASE_1_SUMMARY.md +0 -184
- package/plandocs/PLANNING_SESSION_SUMMARY.md +0 -372
- package/plandocs/PROMPT_OPTIMIZATION_ANALYSIS.md +0 -120
- package/plandocs/PROMPT_SANITY_CHECK.md +0 -120
- package/plandocs/SCRIPT_CLEANUP_FEATURE.md +0 -201
- package/plandocs/SCRIPT_GENERATION_ARCHITECTURE.md +0 -364
- package/plandocs/SELECTOR_IMPROVEMENTS.md +0 -139
- package/plandocs/SESSION_SUMMARY_v0.0.33.md +0 -151
- package/plandocs/TROUBLESHOOTING_SESSION.md +0 -72
- package/plandocs/VISION_DIAGNOSTICS_IMPROVEMENTS.md +0 -336
- package/plandocs/VISUAL_AGENT_EVOLUTION_PLAN.md +0 -396
- package/plandocs/WHATS_NEW_v0.0.33.md +0 -183
- package/src/auth-config.ts +0 -84
- package/src/credit-usage-service.ts +0 -188
- package/src/env-loader.ts +0 -103
- package/src/execution-service.ts +0 -1413
- package/src/file-handler.ts +0 -104
- package/src/index.ts +0 -422
- package/src/llm-facade.ts +0 -821
- package/src/llm-provider.ts +0 -53
- package/src/model-constants.ts +0 -35
- package/src/orchestrator/index.ts +0 -34
- package/src/orchestrator/orchestrator-agent.ts +0 -862
- package/src/orchestrator/orchestrator-agent.ts.backup +0 -1386
- package/src/orchestrator/orchestrator-prompts.ts +0 -474
- package/src/orchestrator/tool-registry.ts +0 -182
- package/src/orchestrator/tools/check-page-ready.ts +0 -75
- package/src/orchestrator/tools/extract-data.ts +0 -92
- package/src/orchestrator/tools/index.ts +0 -12
- package/src/orchestrator/tools/inspect-page.ts +0 -42
- package/src/orchestrator/tools/recall-history.ts +0 -72
- package/src/orchestrator/tools/take-screenshot.ts +0 -128
- package/src/orchestrator/tools/verify-action-result.ts +0 -159
- package/src/orchestrator/types.ts +0 -248
- package/src/playwright-mcp-service.ts +0 -224
- package/src/progress-reporter.ts +0 -144
- package/src/prompts.ts +0 -842
- package/src/providers/backend-proxy-llm-provider.ts +0 -91
- package/src/providers/local-llm-provider.ts +0 -38
- package/src/scenario-service.ts +0 -232
- package/src/scenario-worker-class.ts +0 -1089
- package/src/script-utils.ts +0 -203
- package/src/types.ts +0 -239
- package/src/utils/browser-utils.ts +0 -348
- package/src/utils/coordinate-converter.ts +0 -162
- package/src/utils/page-info-utils.ts +0 -250
- package/testchimp-runner-core-0.0.33.tgz +0 -0
- package/tsconfig.json +0 -19
package/src/execution-service.ts
DELETED
|
@@ -1,1413 +0,0 @@
|
|
|
1
|
-
import { PlaywrightMCPService as PlaywrightService } from './playwright-mcp-service';
|
|
2
|
-
import {
|
|
3
|
-
PlaywrightExecutionRequest,
|
|
4
|
-
PlaywrightExecutionResponse,
|
|
5
|
-
ScriptResult,
|
|
6
|
-
ScriptExecutionRequest,
|
|
7
|
-
ScriptExecutionResponse,
|
|
8
|
-
ScriptStep,
|
|
9
|
-
ExecutionMode,
|
|
10
|
-
StepOperation,
|
|
11
|
-
StepRepairAction
|
|
12
|
-
} from './types';
|
|
13
|
-
import { RepairSuggestionResponse, RepairConfidenceResponse } from './llm-facade';
|
|
14
|
-
import { getEnhancedPageInfo, PageInfo } from './utils/page-info-utils';
|
|
15
|
-
import { initializeBrowser, captureOptimizedScreenshot } from './utils/browser-utils';
|
|
16
|
-
import { LLMFacade } from './llm-facade';
|
|
17
|
-
import { AuthConfig } from './auth-config';
|
|
18
|
-
import { addTestChimpComment } from './script-utils';
|
|
19
|
-
import { CreditUsageService } from './credit-usage-service';
|
|
20
|
-
import { DEFAULT_MODEL, VISION_MODEL } from './model-constants';
|
|
21
|
-
import { LLMProvider } from './llm-provider';
|
|
22
|
-
import { ProgressReporter } from './progress-reporter';
|
|
23
|
-
import { BackendProxyLLMProvider } from './providers/backend-proxy-llm-provider';
|
|
24
|
-
|
|
25
|
-
/**
|
|
26
|
-
* Service for orchestrating Playwright script execution
|
|
27
|
-
*/
|
|
28
|
-
export class ExecutionService {
|
|
29
|
-
private playwrightService: PlaywrightService;
|
|
30
|
-
private llmFacade: LLMFacade;
|
|
31
|
-
private llmProvider: LLMProvider;
|
|
32
|
-
private progressReporter?: ProgressReporter;
|
|
33
|
-
private creditUsageService: CreditUsageService;
|
|
34
|
-
private maxConcurrentExecutions: number;
|
|
35
|
-
private activeExecutions: Set<Promise<any>> = new Set();
|
|
36
|
-
private logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
|
|
37
|
-
|
|
38
|
-
constructor(
|
|
39
|
-
authConfig?: AuthConfig,
|
|
40
|
-
backendUrl?: string,
|
|
41
|
-
maxConcurrentExecutions: number = 10,
|
|
42
|
-
llmProvider?: LLMProvider,
|
|
43
|
-
progressReporter?: ProgressReporter
|
|
44
|
-
) {
|
|
45
|
-
this.playwrightService = new PlaywrightService();
|
|
46
|
-
|
|
47
|
-
// Use provided LLM provider or default to backend proxy (backward compatible)
|
|
48
|
-
this.llmProvider = llmProvider || new BackendProxyLLMProvider(authConfig, backendUrl);
|
|
49
|
-
this.llmFacade = new LLMFacade(this.llmProvider);
|
|
50
|
-
|
|
51
|
-
this.progressReporter = progressReporter;
|
|
52
|
-
this.creditUsageService = new CreditUsageService(authConfig, backendUrl);
|
|
53
|
-
this.maxConcurrentExecutions = maxConcurrentExecutions;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
/**
|
|
57
|
-
* Set a logger callback for capturing execution logs
|
|
58
|
-
*/
|
|
59
|
-
setLogger(logger: (message: string, level?: 'log' | 'error' | 'warn') => void): void {
|
|
60
|
-
this.logger = logger;
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
/**
|
|
64
|
-
* Log a message using the configured logger
|
|
65
|
-
*/
|
|
66
|
-
private log(message: string, level: 'log' | 'error' | 'warn' = 'log'): void {
|
|
67
|
-
if (this.logger) {
|
|
68
|
-
this.logger(message, level);
|
|
69
|
-
}
|
|
70
|
-
// No console fallback - logs are routed to consumer
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
/**
|
|
74
|
-
* Initialize the execution service
|
|
75
|
-
*/
|
|
76
|
-
async initialize(): Promise<void> {
|
|
77
|
-
await this.playwrightService.initialize();
|
|
78
|
-
}
|
|
79
|
-
|
|
80
|
-
/**
|
|
81
|
-
* Set authentication configuration for the service
|
|
82
|
-
* Note: This recreates the LLM provider with new auth config
|
|
83
|
-
*/
|
|
84
|
-
setAuthConfig(authConfig: AuthConfig): void {
|
|
85
|
-
// Recreate LLM provider with new auth config
|
|
86
|
-
this.llmProvider = new BackendProxyLLMProvider(authConfig, undefined);
|
|
87
|
-
this.llmFacade = new LLMFacade(this.llmProvider);
|
|
88
|
-
this.creditUsageService.setAuthConfig(authConfig);
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
/**
|
|
93
|
-
* Execute a script with optional AI repair capabilities
|
|
94
|
-
*/
|
|
95
|
-
async executeScript(request: ScriptExecutionRequest): Promise<ScriptExecutionResponse> {
|
|
96
|
-
// Wait for available slot if at max concurrency
|
|
97
|
-
while (this.activeExecutions.size >= this.maxConcurrentExecutions) {
|
|
98
|
-
await Promise.race(this.activeExecutions);
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
// Create execution promise and track it
|
|
102
|
-
const executionPromise = this.executeScriptInternal(request);
|
|
103
|
-
this.activeExecutions.add(executionPromise);
|
|
104
|
-
|
|
105
|
-
try {
|
|
106
|
-
const result = await executionPromise;
|
|
107
|
-
return result;
|
|
108
|
-
} finally {
|
|
109
|
-
this.activeExecutions.delete(executionPromise);
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
/**
|
|
114
|
-
* Internal script execution method
|
|
115
|
-
*/
|
|
116
|
-
private async executeScriptInternal(request: ScriptExecutionRequest): Promise<ScriptExecutionResponse> {
|
|
117
|
-
const startTime = Date.now();
|
|
118
|
-
const model = request.model || DEFAULT_MODEL;
|
|
119
|
-
|
|
120
|
-
try {
|
|
121
|
-
if (request.mode === ExecutionMode.RUN_EXACTLY) {
|
|
122
|
-
return await this.runExactly(request, startTime);
|
|
123
|
-
} else {
|
|
124
|
-
return await this.runWithAIRepair(request, startTime, model);
|
|
125
|
-
}
|
|
126
|
-
} catch (error) {
|
|
127
|
-
return {
|
|
128
|
-
runStatus: 'failed',
|
|
129
|
-
executionTime: Date.now() - startTime,
|
|
130
|
-
error: error instanceof Error ? error.message : 'Unknown error'
|
|
131
|
-
};
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
/**
|
|
136
|
-
* Execute a complete Playwright test suite as a single job
|
|
137
|
-
*/
|
|
138
|
-
async executeTestSuite(request: PlaywrightExecutionRequest): Promise<PlaywrightExecutionResponse> {
|
|
139
|
-
try {
|
|
140
|
-
// Parse Playwright configuration
|
|
141
|
-
const config = this.parsePlaywrightConfig(request.playwrightConfig);
|
|
142
|
-
|
|
143
|
-
// Execute the entire job (prescript + script + postscript) as one unit
|
|
144
|
-
const jobResult = await this.playwrightService.executeJob(
|
|
145
|
-
request.prescript,
|
|
146
|
-
request.script,
|
|
147
|
-
request.postscript,
|
|
148
|
-
config
|
|
149
|
-
);
|
|
150
|
-
|
|
151
|
-
return {
|
|
152
|
-
success: jobResult.success,
|
|
153
|
-
results: jobResult.results,
|
|
154
|
-
executionTime: jobResult.executionTime,
|
|
155
|
-
error: jobResult.error
|
|
156
|
-
};
|
|
157
|
-
|
|
158
|
-
} catch (error) {
|
|
159
|
-
return {
|
|
160
|
-
success: false,
|
|
161
|
-
results: {
|
|
162
|
-
script: { success: false, output: '', error: '', executionTime: 0 }
|
|
163
|
-
},
|
|
164
|
-
executionTime: 0,
|
|
165
|
-
error: error instanceof Error ? error.message : 'Unknown error occurred'
|
|
166
|
-
};
|
|
167
|
-
}
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
/**
|
|
171
|
-
* Parse Playwright configuration from string
|
|
172
|
-
*/
|
|
173
|
-
private parsePlaywrightConfig(configString: string): any {
|
|
174
|
-
try {
|
|
175
|
-
// Try to parse as JSON first
|
|
176
|
-
const config = JSON.parse(configString);
|
|
177
|
-
return {
|
|
178
|
-
browserType: config.browserType || 'chromium',
|
|
179
|
-
headless: config.headless === true,
|
|
180
|
-
viewport: config.viewport || { width: 1280, height: 720 },
|
|
181
|
-
options: config.options || {}
|
|
182
|
-
};
|
|
183
|
-
} catch {
|
|
184
|
-
// If not JSON, try to extract basic config from JavaScript
|
|
185
|
-
try {
|
|
186
|
-
// Simple regex-based extraction for common config patterns
|
|
187
|
-
const headlessMatch = configString.match(/headless:\s*(true|false)/);
|
|
188
|
-
const viewportMatch = configString.match(/viewport:\s*\{\s*width:\s*(\d+),\s*height:\s*(\d+)\s*\}/);
|
|
189
|
-
const browserMatch = configString.match(/browserType:\s*['"`](chromium|firefox|webkit)['"`]/);
|
|
190
|
-
|
|
191
|
-
return {
|
|
192
|
-
browserType: browserMatch ? browserMatch[1] : 'chromium',
|
|
193
|
-
headless: headlessMatch ? headlessMatch[1] === 'true' : true,
|
|
194
|
-
viewport: viewportMatch ?
|
|
195
|
-
{ width: parseInt(viewportMatch[1]), height: parseInt(viewportMatch[2]) } :
|
|
196
|
-
{ width: 1280, height: 720 },
|
|
197
|
-
options: {}
|
|
198
|
-
};
|
|
199
|
-
} catch {
|
|
200
|
-
// Return default config if parsing fails
|
|
201
|
-
return {
|
|
202
|
-
browserType: 'chromium',
|
|
203
|
-
headless: false,
|
|
204
|
-
viewport: { width: 1280, height: 720 },
|
|
205
|
-
options: {}
|
|
206
|
-
};
|
|
207
|
-
}
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
|
|
211
|
-
/**
|
|
212
|
-
* Close the execution service
|
|
213
|
-
*/
|
|
214
|
-
async close(): Promise<void> {
|
|
215
|
-
await this.playwrightService.close();
|
|
216
|
-
}
|
|
217
|
-
|
|
218
|
-
/**
|
|
219
|
-
* Check if the service is ready
|
|
220
|
-
*/
|
|
221
|
-
isReady(): boolean {
|
|
222
|
-
return this.playwrightService.isReady();
|
|
223
|
-
}
|
|
224
|
-
|
|
225
|
-
private async runExactly(request: ScriptExecutionRequest, startTime: number, model?: string): Promise<ScriptExecutionResponse> {
|
|
226
|
-
// deflakeRunCount: number of deflake attempts (0 means no deflaking, just one attempt)
|
|
227
|
-
const deflakeRunCount = request.deflakeRunCount !== undefined ? request.deflakeRunCount : 0;
|
|
228
|
-
const totalAttempts = deflakeRunCount + 1; // Original run + deflake attempts
|
|
229
|
-
let lastError: Error | null = null;
|
|
230
|
-
|
|
231
|
-
this.log(`runExactly: deflakeRunCount = ${deflakeRunCount}, totalAttempts = ${totalAttempts}`);
|
|
232
|
-
|
|
233
|
-
// Script content should be provided by the caller (TestChimpService)
|
|
234
|
-
// The TestChimpService handles file reading through the appropriate FileHandler
|
|
235
|
-
if (!request.script) {
|
|
236
|
-
throw new Error('Script content is required for execution. The TestChimpService should read the file and provide script content.');
|
|
237
|
-
}
|
|
238
|
-
|
|
239
|
-
// Check if we should use existing browser or create new one
|
|
240
|
-
const useExistingBrowser = !!(request.existingBrowser && request.existingContext && request.existingPage);
|
|
241
|
-
|
|
242
|
-
if (useExistingBrowser) {
|
|
243
|
-
this.log('Using existing browser/page provided by caller - single attempt only (no internal deflaking)');
|
|
244
|
-
// Single attempt with existing browser (caller handles deflaking by creating fresh browsers)
|
|
245
|
-
const browser = request.existingBrowser;
|
|
246
|
-
const context = request.existingContext;
|
|
247
|
-
const page = request.existingPage;
|
|
248
|
-
|
|
249
|
-
try {
|
|
250
|
-
// LIFECYCLE: Call beforeStartTest if provided
|
|
251
|
-
if (this.progressReporter?.beforeStartTest) {
|
|
252
|
-
await this.progressReporter.beforeStartTest(page, browser, context);
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
// Execute the script as-is
|
|
256
|
-
await this.executeScriptContent(request.script, page);
|
|
257
|
-
|
|
258
|
-
// LIFECYCLE: Call afterEndTest on success
|
|
259
|
-
if (this.progressReporter?.afterEndTest) {
|
|
260
|
-
await this.progressReporter.afterEndTest('passed', undefined, page);
|
|
261
|
-
}
|
|
262
|
-
|
|
263
|
-
// Don't close browser - caller owns it
|
|
264
|
-
|
|
265
|
-
return {
|
|
266
|
-
runStatus: 'success',
|
|
267
|
-
numDeflakeRuns: 0,
|
|
268
|
-
executionTime: Date.now() - startTime
|
|
269
|
-
};
|
|
270
|
-
} catch (error) {
|
|
271
|
-
lastError = error instanceof Error ? error : new Error('Script execution failed');
|
|
272
|
-
this.log(`Execution failed: ${lastError.message}`);
|
|
273
|
-
|
|
274
|
-
// LIFECYCLE: Call afterEndTest on failure
|
|
275
|
-
if (this.progressReporter?.afterEndTest) {
|
|
276
|
-
try {
|
|
277
|
-
await this.progressReporter.afterEndTest('failed', lastError.message, page);
|
|
278
|
-
} catch (callbackError) {
|
|
279
|
-
this.log(`afterEndTest callback failed: ${callbackError}`, 'warn');
|
|
280
|
-
}
|
|
281
|
-
}
|
|
282
|
-
|
|
283
|
-
return {
|
|
284
|
-
runStatus: 'failed',
|
|
285
|
-
numDeflakeRuns: 0,
|
|
286
|
-
executionTime: Date.now() - startTime,
|
|
287
|
-
error: lastError.message
|
|
288
|
-
};
|
|
289
|
-
}
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
// Create our own browser (original behavior)
|
|
293
|
-
for (let attempt = 1; attempt <= totalAttempts; attempt++) {
|
|
294
|
-
this.log(`Attempting deflake run ${attempt}/${totalAttempts}`);
|
|
295
|
-
const { browser, context, page } = await this.initializeBrowser(request.playwrightConfig, request.headless, request.playwrightConfigFilePath);
|
|
296
|
-
|
|
297
|
-
try {
|
|
298
|
-
// LIFECYCLE: Call beforeStartTest if provided
|
|
299
|
-
if (this.progressReporter?.beforeStartTest) {
|
|
300
|
-
await this.progressReporter.beforeStartTest(page, browser, context);
|
|
301
|
-
}
|
|
302
|
-
|
|
303
|
-
// Execute the script as-is
|
|
304
|
-
await this.executeScriptContent(request.script, page);
|
|
305
|
-
|
|
306
|
-
// LIFECYCLE: Call afterEndTest on success
|
|
307
|
-
if (this.progressReporter?.afterEndTest) {
|
|
308
|
-
await this.progressReporter.afterEndTest('passed', undefined, page);
|
|
309
|
-
}
|
|
310
|
-
|
|
311
|
-
await browser.close();
|
|
312
|
-
|
|
313
|
-
// Success! Return immediately
|
|
314
|
-
return {
|
|
315
|
-
runStatus: 'success',
|
|
316
|
-
numDeflakeRuns: attempt - 1, // Count only deflaking runs (exclude original run)
|
|
317
|
-
executionTime: Date.now() - startTime
|
|
318
|
-
};
|
|
319
|
-
} catch (error) {
|
|
320
|
-
lastError = error instanceof Error ? error : new Error('Script execution failed');
|
|
321
|
-
this.log(`Initial run failed: ${lastError.message}`);
|
|
322
|
-
|
|
323
|
-
// LIFECYCLE: Call afterEndTest on failure
|
|
324
|
-
if (this.progressReporter?.afterEndTest) {
|
|
325
|
-
try {
|
|
326
|
-
await this.progressReporter.afterEndTest('failed', lastError.message, page);
|
|
327
|
-
} catch (callbackError) {
|
|
328
|
-
this.log(`afterEndTest callback failed: ${callbackError}`, 'warn');
|
|
329
|
-
}
|
|
330
|
-
}
|
|
331
|
-
|
|
332
|
-
try {
|
|
333
|
-
await browser.close();
|
|
334
|
-
} catch (closeError) {
|
|
335
|
-
// Browser might already be closed
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
// If this is not the last attempt, continue to next attempt
|
|
339
|
-
if (attempt < totalAttempts) {
|
|
340
|
-
this.log(`Deflaking attempt ${attempt} failed, trying again... (${attempt + 1}/${totalAttempts})`);
|
|
341
|
-
continue;
|
|
342
|
-
}
|
|
343
|
-
}
|
|
344
|
-
}
|
|
345
|
-
|
|
346
|
-
// All attempts failed
|
|
347
|
-
return {
|
|
348
|
-
runStatus: 'failed',
|
|
349
|
-
numDeflakeRuns: deflakeRunCount, // Count only deflaking runs (exclude original run)
|
|
350
|
-
executionTime: Date.now() - startTime,
|
|
351
|
-
error: lastError?.message || 'All deflaking attempts failed'
|
|
352
|
-
};
|
|
353
|
-
}
|
|
354
|
-
|
|
355
|
-
private async runWithAIRepair(request: ScriptExecutionRequest, startTime: number, model: string): Promise<ScriptExecutionResponse> {
|
|
356
|
-
const repairFlexibility = request.repairFlexibility || 3;
|
|
357
|
-
const attemptRunExactlyFirst = request.attemptRunExactlyFirst || false;
|
|
358
|
-
|
|
359
|
-
// Script content is required UNLESS pre-parsed steps are provided
|
|
360
|
-
if (!request.script && (!request.steps || request.steps.length === 0)) {
|
|
361
|
-
throw new Error('Script content is required for AI repair. The TestChimpService should read the file and provide script content.');
|
|
362
|
-
}
|
|
363
|
-
|
|
364
|
-
// Check if we should use existing browser
|
|
365
|
-
const useExistingBrowser = !!(request.existingBrowser && request.existingContext && request.existingPage);
|
|
366
|
-
|
|
367
|
-
// Optionally try runExactly first (with deflaking if specified)
|
|
368
|
-
if (attemptRunExactlyFirst) {
|
|
369
|
-
const deflakeCount = request.deflakeRunCount || 0;
|
|
370
|
-
this.log(`Attempting runExactly first with ${deflakeCount} deflake attempts...`);
|
|
371
|
-
const runExactlyResult = await this.runExactly({
|
|
372
|
-
...request,
|
|
373
|
-
mode: ExecutionMode.RUN_EXACTLY,
|
|
374
|
-
deflakeRunCount: deflakeCount
|
|
375
|
-
}, startTime, model);
|
|
376
|
-
|
|
377
|
-
if (runExactlyResult.runStatus === 'success') {
|
|
378
|
-
this.log('runExactly succeeded, returning without AI repair');
|
|
379
|
-
return runExactlyResult;
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
this.log('runExactly failed, proceeding with AI repair...');
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
// Start AI repair process
|
|
386
|
-
this.log('Starting AI repair process...');
|
|
387
|
-
|
|
388
|
-
try {
|
|
389
|
-
let repairBrowser, repairContext, repairPage, steps, updatedSteps;
|
|
390
|
-
|
|
391
|
-
if (useExistingBrowser) {
|
|
392
|
-
// Use existing browser
|
|
393
|
-
this.log('Using existing browser for AI repair...');
|
|
394
|
-
repairBrowser = request.existingBrowser;
|
|
395
|
-
repairContext = request.existingContext;
|
|
396
|
-
repairPage = request.existingPage;
|
|
397
|
-
|
|
398
|
-
// Use pre-parsed steps if provided (preserves step IDs from canonical tree),
|
|
399
|
-
// otherwise parse script into steps
|
|
400
|
-
if (request.steps && request.steps.length > 0) {
|
|
401
|
-
this.log(`Using ${request.steps.length} pre-parsed steps (IDs preserved)`);
|
|
402
|
-
steps = request.steps;
|
|
403
|
-
} else {
|
|
404
|
-
this.log('Parsing script into steps...');
|
|
405
|
-
if (!request.script) {
|
|
406
|
-
throw new Error('Script is required when steps are not provided');
|
|
407
|
-
}
|
|
408
|
-
steps = await this.parseScriptIntoSteps(request.script, model);
|
|
409
|
-
}
|
|
410
|
-
|
|
411
|
-
// LIFECYCLE: Call beforeStartTest if provided
|
|
412
|
-
if (this.progressReporter?.beforeStartTest) {
|
|
413
|
-
await this.progressReporter.beforeStartTest(repairPage, repairBrowser, repairContext);
|
|
414
|
-
}
|
|
415
|
-
|
|
416
|
-
this.log('Starting AI repair with parsed steps...');
|
|
417
|
-
updatedSteps = await this.repairStepsWithAI(steps, repairPage, repairFlexibility, model, request.jobId);
|
|
418
|
-
} else {
|
|
419
|
-
// Use pre-parsed steps if provided, otherwise parse script
|
|
420
|
-
if (request.steps && request.steps.length > 0) {
|
|
421
|
-
this.log(`Using ${request.steps.length} pre-parsed steps (IDs preserved)`);
|
|
422
|
-
this.log('Initializing repair browser...');
|
|
423
|
-
steps = request.steps;
|
|
424
|
-
const browserInstance = await this.initializeBrowser(request.playwrightConfig, request.headless, request.playwrightConfigFilePath);
|
|
425
|
-
repairBrowser = browserInstance.browser;
|
|
426
|
-
repairContext = browserInstance.context;
|
|
427
|
-
repairPage = browserInstance.page;
|
|
428
|
-
} else {
|
|
429
|
-
// Start browser initialization and script parsing in parallel for faster startup
|
|
430
|
-
this.log('Initializing repair browser and parsing script...');
|
|
431
|
-
if (!request.script) {
|
|
432
|
-
throw new Error('Script is required when steps are not provided');
|
|
433
|
-
}
|
|
434
|
-
const results = await Promise.all([
|
|
435
|
-
this.parseScriptIntoSteps(request.script, model),
|
|
436
|
-
this.initializeBrowser(request.playwrightConfig, request.headless, request.playwrightConfigFilePath) // Use request.headless (defaults to false/headed)
|
|
437
|
-
]);
|
|
438
|
-
|
|
439
|
-
steps = results[0];
|
|
440
|
-
repairBrowser = results[1].browser;
|
|
441
|
-
repairContext = results[1].context;
|
|
442
|
-
repairPage = results[1].page;
|
|
443
|
-
}
|
|
444
|
-
|
|
445
|
-
// LIFECYCLE: Call beforeStartTest if provided
|
|
446
|
-
if (this.progressReporter?.beforeStartTest) {
|
|
447
|
-
await this.progressReporter.beforeStartTest(repairPage, repairBrowser, repairContext);
|
|
448
|
-
}
|
|
449
|
-
|
|
450
|
-
this.log('Starting AI repair with parsed steps...');
|
|
451
|
-
updatedSteps = await this.repairStepsWithAI(steps, repairPage, repairFlexibility, model, request.jobId);
|
|
452
|
-
}
|
|
453
|
-
|
|
454
|
-
// Always generate the updated script
|
|
455
|
-
const updatedScript = this.generateUpdatedScript(updatedSteps);
|
|
456
|
-
|
|
457
|
-
// Check if repair was successful by seeing if we completed all steps
|
|
458
|
-
const allStepsSuccessful = updatedSteps.length > 0 && updatedSteps.every(step => step.success);
|
|
459
|
-
|
|
460
|
-
// Check if we have any successful repairs (partial success)
|
|
461
|
-
const hasSuccessfulRepairs = updatedSteps.some(step => step.success);
|
|
462
|
-
|
|
463
|
-
// Debug: Log step success status
|
|
464
|
-
this.log('Step success status: ' + updatedSteps.map((step, index) => `Step ${index + 1}: ${step.success ? 'SUCCESS' : 'FAILED'}`).join(', '));
|
|
465
|
-
this.log(`All steps successful: ${allStepsSuccessful}`);
|
|
466
|
-
this.log(`Has successful repairs: ${hasSuccessfulRepairs}`);
|
|
467
|
-
|
|
468
|
-
// Debug: Log individual step details
|
|
469
|
-
updatedSteps.forEach((step, index) => {
|
|
470
|
-
this.log(`Step ${index + 1} details: success=${step.success}, description="${step.description}"`);
|
|
471
|
-
});
|
|
472
|
-
|
|
473
|
-
// Update file if we have any successful repairs (partial or complete)
|
|
474
|
-
if (hasSuccessfulRepairs) {
|
|
475
|
-
const confidenceResponse = await this.llmFacade.assessRepairConfidence(request.script!, updatedScript, model);
|
|
476
|
-
const finalScript = await this.llmFacade.generateFinalScript(request.script!, updatedScript, confidenceResponse.advice, model);
|
|
477
|
-
|
|
478
|
-
// Ensure the final script has the correct TestChimp comment format with repair advice
|
|
479
|
-
const scriptWithRepairAdvice = addTestChimpComment(finalScript, confidenceResponse.advice);
|
|
480
|
-
|
|
481
|
-
// Report credit usage for successful AI repair
|
|
482
|
-
this.creditUsageService.reportAIRepairCredit().catch(error => {
|
|
483
|
-
this.log(`Failed to report credit usage for AI repair: ${error}`, 'warn');
|
|
484
|
-
});
|
|
485
|
-
|
|
486
|
-
// LIFECYCLE: Call afterEndTest (partial or complete success)
|
|
487
|
-
if (this.progressReporter?.afterEndTest) {
|
|
488
|
-
try {
|
|
489
|
-
await this.progressReporter.afterEndTest(
|
|
490
|
-
allStepsSuccessful ? 'passed' : 'failed',
|
|
491
|
-
allStepsSuccessful ? undefined : 'Partial repair success',
|
|
492
|
-
repairPage
|
|
493
|
-
);
|
|
494
|
-
} catch (callbackError) {
|
|
495
|
-
this.log(`afterEndTest callback failed: ${callbackError}`, 'warn');
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
// Only close browser if we created it (not provided by caller)
|
|
500
|
-
if (!useExistingBrowser) {
|
|
501
|
-
await repairBrowser.close();
|
|
502
|
-
}
|
|
503
|
-
|
|
504
|
-
return {
|
|
505
|
-
runStatus: 'failed', // Original script failed
|
|
506
|
-
repairStatus: allStepsSuccessful ? 'success' : 'partial', // Complete or partial repair success
|
|
507
|
-
repairConfidence: confidenceResponse.confidence,
|
|
508
|
-
repairAdvice: confidenceResponse.advice,
|
|
509
|
-
updatedScript: scriptWithRepairAdvice, // Return the drop-in replacement script with proper TestChimp comment
|
|
510
|
-
numDeflakeRuns: 0, // Deflaking is handled by caller before calling runWithAIRepair
|
|
511
|
-
executionTime: Date.now() - startTime
|
|
512
|
-
};
|
|
513
|
-
} else {
|
|
514
|
-
// No successful repairs at all
|
|
515
|
-
|
|
516
|
-
// LIFECYCLE: Call afterEndTest (complete failure)
|
|
517
|
-
if (this.progressReporter?.afterEndTest) {
|
|
518
|
-
try {
|
|
519
|
-
await this.progressReporter.afterEndTest('failed', 'AI repair could not fix any steps', repairPage);
|
|
520
|
-
} catch (callbackError) {
|
|
521
|
-
this.log(`afterEndTest callback failed: ${callbackError}`, 'warn');
|
|
522
|
-
}
|
|
523
|
-
}
|
|
524
|
-
|
|
525
|
-
// Only close browser if we created it (not provided by caller)
|
|
526
|
-
if (!useExistingBrowser) {
|
|
527
|
-
await repairBrowser.close();
|
|
528
|
-
}
|
|
529
|
-
|
|
530
|
-
return {
|
|
531
|
-
runStatus: 'failed', // Original script failed
|
|
532
|
-
repairStatus: 'failed',
|
|
533
|
-
repairConfidence: 0,
|
|
534
|
-
repairAdvice: 'AI repair could not fix any steps',
|
|
535
|
-
updatedScript: request.script!, // Return original script since no repairs were successful
|
|
536
|
-
numDeflakeRuns: 0, // Deflaking is handled by caller before calling runWithAIRepair
|
|
537
|
-
executionTime: Date.now() - startTime,
|
|
538
|
-
error: 'AI repair could not fix any steps'
|
|
539
|
-
};
|
|
540
|
-
}
|
|
541
|
-
} catch (error) {
|
|
542
|
-
return {
|
|
543
|
-
runStatus: 'failed',
|
|
544
|
-
repairStatus: 'failed',
|
|
545
|
-
numDeflakeRuns: 0, // Deflaking is handled by caller before calling runWithAIRepair
|
|
546
|
-
executionTime: Date.now() - startTime,
|
|
547
|
-
error: error instanceof Error ? error.message : 'Script execution failed'
|
|
548
|
-
};
|
|
549
|
-
}
|
|
550
|
-
}
|
|
551
|
-
|
|
552
|
-
private async parseScriptIntoSteps(script: string, model: string): Promise<(ScriptStep & { success?: boolean; error?: string })[]> {
|
|
553
|
-
// First try LLM-based parsing
|
|
554
|
-
try {
|
|
555
|
-
this.log('Attempting LLM-based script parsing...');
|
|
556
|
-
const result = await this.llmFacade.parseScriptIntoSteps(script, model);
|
|
557
|
-
this.log(`LLM parsing successful, got ${result.length} steps`);
|
|
558
|
-
return result;
|
|
559
|
-
} catch (error) {
|
|
560
|
-
this.log(`LLM parsing failed, falling back to code parsing: ${error}`);
|
|
561
|
-
const fallbackResult = this.parseScriptIntoStepsFallback(script);
|
|
562
|
-
this.log(`Fallback parsing successful, got ${fallbackResult.length} steps`);
|
|
563
|
-
return fallbackResult;
|
|
564
|
-
}
|
|
565
|
-
}
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
private parseScriptIntoStepsFallback(script: string): (ScriptStep & { success?: boolean; error?: string })[] {
|
|
569
|
-
const lines = script.split('\n');
|
|
570
|
-
const steps: (ScriptStep & { success?: boolean; error?: string })[] = [];
|
|
571
|
-
let currentStep: ScriptStep | null = null;
|
|
572
|
-
let currentCode: string[] = [];
|
|
573
|
-
|
|
574
|
-
for (const line of lines) {
|
|
575
|
-
const trimmedLine = line.trim();
|
|
576
|
-
|
|
577
|
-
// Check for step comment
|
|
578
|
-
if (trimmedLine.startsWith('// Step ')) {
|
|
579
|
-
// Save previous step if exists and has code
|
|
580
|
-
if (currentStep) {
|
|
581
|
-
const code = currentCode.join('\n').trim();
|
|
582
|
-
const cleanedCode = this.cleanStepCode(code);
|
|
583
|
-
if (cleanedCode) {
|
|
584
|
-
currentStep.code = cleanedCode;
|
|
585
|
-
steps.push(currentStep);
|
|
586
|
-
}
|
|
587
|
-
}
|
|
588
|
-
|
|
589
|
-
// Start new step
|
|
590
|
-
const description = trimmedLine.replace(/^\/\/\s*Step\s*\d+:\s*/, '').replace(/\s*\[FAILED\]\s*$/, '').trim();
|
|
591
|
-
currentStep = { description, code: '' };
|
|
592
|
-
currentCode = [];
|
|
593
|
-
} else if (trimmedLine && !trimmedLine.startsWith('import') && !trimmedLine.startsWith('test(') && !trimmedLine.startsWith('});')) {
|
|
594
|
-
// Add code line to current step
|
|
595
|
-
if (currentStep) {
|
|
596
|
-
currentCode.push(line);
|
|
597
|
-
}
|
|
598
|
-
}
|
|
599
|
-
}
|
|
600
|
-
|
|
601
|
-
// Add the last step if it has code
|
|
602
|
-
if (currentStep) {
|
|
603
|
-
const code = currentCode.join('\n').trim();
|
|
604
|
-
const cleanedCode = this.cleanStepCode(code);
|
|
605
|
-
if (cleanedCode) {
|
|
606
|
-
currentStep.code = cleanedCode;
|
|
607
|
-
steps.push(currentStep);
|
|
608
|
-
}
|
|
609
|
-
}
|
|
610
|
-
|
|
611
|
-
return steps;
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
private async repairStepsWithAI(
|
|
615
|
-
steps: (ScriptStep & { success?: boolean; error?: string })[],
|
|
616
|
-
page: any,
|
|
617
|
-
repairFlexibility: number,
|
|
618
|
-
model: string,
|
|
619
|
-
jobId?: string
|
|
620
|
-
): Promise<(ScriptStep & { success?: boolean; error?: string })[]> {
|
|
621
|
-
let updatedSteps = [...steps];
|
|
622
|
-
const maxTries = 3;
|
|
623
|
-
const recentRepairs: Array<{
|
|
624
|
-
stepNumber: number;
|
|
625
|
-
operation: string;
|
|
626
|
-
originalDescription?: string;
|
|
627
|
-
newDescription?: string;
|
|
628
|
-
originalCode?: string;
|
|
629
|
-
newCode?: string;
|
|
630
|
-
}> = [];
|
|
631
|
-
|
|
632
|
-
// Create a shared execution context that accumulates all executed code for variable tracking
|
|
633
|
-
let executionContext = '';
|
|
634
|
-
const contextVariables = new Map<string, any>();
|
|
635
|
-
|
|
636
|
-
let i = 0;
|
|
637
|
-
while (i < updatedSteps.length) {
|
|
638
|
-
const step = updatedSteps[i];
|
|
639
|
-
this.log(`Loop iteration: i=${i}, step description="${step.description}", total steps=${updatedSteps.length}`);
|
|
640
|
-
|
|
641
|
-
try {
|
|
642
|
-
// LIFECYCLE: Call beforeStepStart if provided
|
|
643
|
-
if (this.progressReporter?.beforeStepStart) {
|
|
644
|
-
await this.progressReporter.beforeStepStart(
|
|
645
|
-
{
|
|
646
|
-
stepId: step.id, // Preserve original step ID if provided
|
|
647
|
-
stepNumber: i + 1,
|
|
648
|
-
description: step.description,
|
|
649
|
-
code: step.code
|
|
650
|
-
},
|
|
651
|
-
page
|
|
652
|
-
);
|
|
653
|
-
}
|
|
654
|
-
|
|
655
|
-
// Try to execute the step directly without context replay
|
|
656
|
-
this.log(`Attempting Step ${i + 1}: ${step.description}`);
|
|
657
|
-
this.log(` Code: ${step.code}`);
|
|
658
|
-
await this.executeStepCode(step.code, page);
|
|
659
|
-
step.success = true;
|
|
660
|
-
this.log(`Step ${i + 1} executed successfully: ${step.description}`);
|
|
661
|
-
this.log(`Step ${i + 1} success status set to: ${step.success}`);
|
|
662
|
-
|
|
663
|
-
// Report successful step execution
|
|
664
|
-
this.log(`DEBUG: About to check callback - progressReporter=${!!this.progressReporter}, onStepProgress=${!!this.progressReporter?.onStepProgress}, jobId=${jobId}`);
|
|
665
|
-
if (this.progressReporter?.onStepProgress && jobId) {
|
|
666
|
-
this.log(`DEBUG: Firing onStepProgress callback for step ${i + 1}, stepId=${step.id}`);
|
|
667
|
-
await this.progressReporter.onStepProgress({
|
|
668
|
-
jobId,
|
|
669
|
-
stepId: step.id, // Preserve original step ID if provided
|
|
670
|
-
stepNumber: i + 1,
|
|
671
|
-
description: step.description,
|
|
672
|
-
code: step.code,
|
|
673
|
-
status: 'SUCCESS_STEP_EXECUTION' as any,
|
|
674
|
-
wasRepaired: false
|
|
675
|
-
});
|
|
676
|
-
this.log(`DEBUG: onStepProgress callback completed for step ${i + 1}`);
|
|
677
|
-
} else {
|
|
678
|
-
this.log(`DEBUG: Skipping callback - conditions not met`);
|
|
679
|
-
}
|
|
680
|
-
|
|
681
|
-
// Add this step's code to the execution context for future steps (for variable tracking)
|
|
682
|
-
executionContext += step.code + '\n';
|
|
683
|
-
i++; // Move to next step
|
|
684
|
-
} catch (error) {
|
|
685
|
-
this.log(`Step ${i + 1} failed: ${step.description}`);
|
|
686
|
-
this.log(` Failed code: ${step.code}`);
|
|
687
|
-
this.log(` Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
|
|
688
|
-
if (error instanceof Error && error.stack) {
|
|
689
|
-
this.log(` Stack trace: ${error.stack}`);
|
|
690
|
-
}
|
|
691
|
-
step.success = false;
|
|
692
|
-
step.error = this.safeSerializeError(error);
|
|
693
|
-
|
|
694
|
-
// Try multiple repair attempts
|
|
695
|
-
const repairHistory: Array<{
|
|
696
|
-
attempt: number;
|
|
697
|
-
action: StepRepairAction;
|
|
698
|
-
error: string;
|
|
699
|
-
pageInfo: PageInfo;
|
|
700
|
-
}> = [];
|
|
701
|
-
|
|
702
|
-
let repairSuccess = false;
|
|
703
|
-
const originalDescription = step.description;
|
|
704
|
-
const originalCode = step.code;
|
|
705
|
-
let usedVisionMode = false;
|
|
706
|
-
|
|
707
|
-
for (let attempt = 1; attempt <= maxTries; attempt++) {
|
|
708
|
-
this.log(`Step ${i + 1} repair attempt ${attempt}/${maxTries}`);
|
|
709
|
-
|
|
710
|
-
// Get current page state for AI repair
|
|
711
|
-
const pageInfo = await this.getEnhancedPageInfo(page);
|
|
712
|
-
|
|
713
|
-
// Build failure history for LLM context
|
|
714
|
-
const failureHistory = this.buildFailureHistory(repairHistory, step, error);
|
|
715
|
-
|
|
716
|
-
// Build recent repairs context for LLM
|
|
717
|
-
const recentRepairsContext = this.buildRecentRepairsContext(recentRepairs);
|
|
718
|
-
|
|
719
|
-
let repairSuggestion;
|
|
720
|
-
|
|
721
|
-
// VISION-BASED FALLBACK: After 2 regular repair attempts, consider vision diagnostics on final attempt
|
|
722
|
-
if (attempt === maxTries - 1 && repairHistory.length >= 2 && !usedVisionMode) {
|
|
723
|
-
// Ask LLM if screenshot would help for repair diagnostics
|
|
724
|
-
this.log(` 🤔 After ${repairHistory.length} failed repairs: Asking LLM if screenshot would help (last resort)...`);
|
|
725
|
-
|
|
726
|
-
const screenshotNeed = await this.llmFacade.assessScreenshotNeed(
|
|
727
|
-
step.description,
|
|
728
|
-
step.error || 'Unknown error',
|
|
729
|
-
repairHistory.length + 1,
|
|
730
|
-
pageInfo,
|
|
731
|
-
model
|
|
732
|
-
);
|
|
733
|
-
|
|
734
|
-
this.log(` 💭 LLM assessment: ${screenshotNeed.needsScreenshot ? 'SCREENSHOT NEEDED' : 'NO SCREENSHOT'} - ${screenshotNeed.reason}`);
|
|
735
|
-
|
|
736
|
-
if (screenshotNeed.needsScreenshot) {
|
|
737
|
-
// Two-step supervisor pattern for vision-based repair:
|
|
738
|
-
// 1. Supervisor analyzes screenshot and provides diagnostic insights
|
|
739
|
-
// 2. Get repair suggestion with enhanced context from vision analysis
|
|
740
|
-
|
|
741
|
-
this.log(` 📸 Taking screenshot for supervisor analysis...`);
|
|
742
|
-
|
|
743
|
-
// Capture optimized screenshot using utility method
|
|
744
|
-
const imageDataUrl = await captureOptimizedScreenshot(
|
|
745
|
-
page,
|
|
746
|
-
{ timeout: 10000 }, // Uses default quality 60
|
|
747
|
-
(msg) => this.log(msg)
|
|
748
|
-
);
|
|
749
|
-
|
|
750
|
-
this.log(` 👔 STEP 1: Supervisor analyzing screenshot (${VISION_MODEL})...`);
|
|
751
|
-
const supervisorDiagnostics = await this.llmFacade.getVisionDiagnostics(
|
|
752
|
-
step.description,
|
|
753
|
-
pageInfo,
|
|
754
|
-
[], // No previous steps context for repair
|
|
755
|
-
step.error,
|
|
756
|
-
imageDataUrl,
|
|
757
|
-
VISION_MODEL
|
|
758
|
-
);
|
|
759
|
-
|
|
760
|
-
// DEBUG: Log vision diagnostics
|
|
761
|
-
this.log(` 📸 Visual insights: ${supervisorDiagnostics.visualAnalysis}`);
|
|
762
|
-
this.log(` 🔍 Root cause: ${supervisorDiagnostics.rootCause}`);
|
|
763
|
-
this.log(` 💡 Recommended approach: ${supervisorDiagnostics.recommendedApproach}`);
|
|
764
|
-
if (supervisorDiagnostics.elementsFound.length > 0) {
|
|
765
|
-
this.log(` ✅ Elements found: ${supervisorDiagnostics.elementsFound.join(', ')}`);
|
|
766
|
-
}
|
|
767
|
-
if (supervisorDiagnostics.elementsNotFound.length > 0) {
|
|
768
|
-
this.log(` ❌ Elements not found: ${supervisorDiagnostics.elementsNotFound.join(', ')}`);
|
|
769
|
-
}
|
|
770
|
-
|
|
771
|
-
// Get repair suggestion with vision-enhanced context
|
|
772
|
-
this.log(` 🔨 STEP 2: Getting repair suggestion with vision insights...`);
|
|
773
|
-
const visionEnhancedFailureHistory = `${failureHistory}
|
|
774
|
-
|
|
775
|
-
VISION-BASED DIAGNOSTIC INSIGHTS:
|
|
776
|
-
Visual Analysis: ${supervisorDiagnostics.visualAnalysis}
|
|
777
|
-
Root Cause: ${supervisorDiagnostics.rootCause}
|
|
778
|
-
Recommended Approach: ${supervisorDiagnostics.recommendedApproach}
|
|
779
|
-
Elements Found: ${supervisorDiagnostics.elementsFound.join(', ') || 'None'}
|
|
780
|
-
Elements Not Found: ${supervisorDiagnostics.elementsNotFound.join(', ') || 'None'}
|
|
781
|
-
|
|
782
|
-
Use these vision insights to inform your repair strategy.`;
|
|
783
|
-
|
|
784
|
-
repairSuggestion = await this.llmFacade.getRepairSuggestion(
|
|
785
|
-
step.description,
|
|
786
|
-
step.code,
|
|
787
|
-
step.error || 'Unknown error',
|
|
788
|
-
pageInfo,
|
|
789
|
-
visionEnhancedFailureHistory,
|
|
790
|
-
recentRepairsContext,
|
|
791
|
-
model
|
|
792
|
-
);
|
|
793
|
-
|
|
794
|
-
usedVisionMode = true;
|
|
795
|
-
} else {
|
|
796
|
-
// Regular repair without vision
|
|
797
|
-
if (screenshotNeed.alternativeApproach) {
|
|
798
|
-
this.log(` 💡 Alternative approach: ${screenshotNeed.alternativeApproach}`);
|
|
799
|
-
}
|
|
800
|
-
repairSuggestion = await this.llmFacade.getRepairSuggestion(
|
|
801
|
-
step.description,
|
|
802
|
-
step.code,
|
|
803
|
-
step.error || 'Unknown error',
|
|
804
|
-
pageInfo,
|
|
805
|
-
failureHistory,
|
|
806
|
-
recentRepairsContext,
|
|
807
|
-
model
|
|
808
|
-
);
|
|
809
|
-
}
|
|
810
|
-
} else {
|
|
811
|
-
// Regular repair attempt (first 2 attempts or already used vision)
|
|
812
|
-
repairSuggestion = await this.llmFacade.getRepairSuggestion(
|
|
813
|
-
step.description,
|
|
814
|
-
step.code,
|
|
815
|
-
step.error || 'Unknown error',
|
|
816
|
-
pageInfo,
|
|
817
|
-
failureHistory,
|
|
818
|
-
recentRepairsContext,
|
|
819
|
-
model
|
|
820
|
-
);
|
|
821
|
-
}
|
|
822
|
-
|
|
823
|
-
if (!repairSuggestion.shouldContinue) {
|
|
824
|
-
this.log(`AI decided to stop repair at attempt ${attempt}: ${repairSuggestion.reason}`);
|
|
825
|
-
break;
|
|
826
|
-
}
|
|
827
|
-
|
|
828
|
-
// Apply the repair action
|
|
829
|
-
try {
|
|
830
|
-
// Set the step index and insertAfterIndex on the client side based on current step being processed
|
|
831
|
-
const repairAction = {
|
|
832
|
-
...repairSuggestion.action,
|
|
833
|
-
stepIndex: i, // Client-side step index management
|
|
834
|
-
insertAfterIndex: repairSuggestion.action.operation === StepOperation.INSERT ? i - 1 : undefined // For INSERT, insert before current step
|
|
835
|
-
};
|
|
836
|
-
|
|
837
|
-
this.log(`🔧 Applying repair action: ${repairAction.operation} on step ${repairAction.stepIndex}`);
|
|
838
|
-
this.log(`🔧 Steps array before repair: ${updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`).join(', ')}`);
|
|
839
|
-
|
|
840
|
-
const result = await this.applyRepairActionInContext(repairAction, updatedSteps, i, page, executionContext, contextVariables);
|
|
841
|
-
|
|
842
|
-
if (result.success) {
|
|
843
|
-
repairSuccess = true;
|
|
844
|
-
this.log(`🔧 Steps array after repair: ${updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`).join(', ')}`);
|
|
845
|
-
|
|
846
|
-
// Mark the appropriate step(s) as successful based on operation type
|
|
847
|
-
if (repairAction.operation === StepOperation.MODIFY) {
|
|
848
|
-
// For MODIFY: mark the modified step as successful
|
|
849
|
-
step.success = true;
|
|
850
|
-
step.error = undefined;
|
|
851
|
-
updatedSteps[i].success = true;
|
|
852
|
-
updatedSteps[i].error = undefined;
|
|
853
|
-
this.log(`Step ${i + 1} marked as successful after MODIFY repair`);
|
|
854
|
-
|
|
855
|
-
// Report repaired step
|
|
856
|
-
if (this.progressReporter?.onStepProgress && jobId) {
|
|
857
|
-
this.log(`DEBUG: Reporting repaired step ${i + 1}:`);
|
|
858
|
-
this.log(` description: ${updatedSteps[i].description}`);
|
|
859
|
-
this.log(` code: ${updatedSteps[i].code}`);
|
|
860
|
-
await this.progressReporter.onStepProgress({
|
|
861
|
-
jobId,
|
|
862
|
-
stepId: updatedSteps[i].id, // Preserve original step ID if provided
|
|
863
|
-
stepNumber: i + 1,
|
|
864
|
-
description: updatedSteps[i].description,
|
|
865
|
-
code: updatedSteps[i].code,
|
|
866
|
-
status: 'SUCCESS_STEP_EXECUTION' as any,
|
|
867
|
-
wasRepaired: true
|
|
868
|
-
});
|
|
869
|
-
}
|
|
870
|
-
} else if (repairAction.operation === StepOperation.INSERT) {
|
|
871
|
-
// For INSERT: mark the newly inserted step as successful
|
|
872
|
-
const insertIndex = repairAction.insertAfterIndex !== undefined ? repairAction.insertAfterIndex + 1 : i + 1;
|
|
873
|
-
if (updatedSteps[insertIndex]) {
|
|
874
|
-
updatedSteps[insertIndex].success = true;
|
|
875
|
-
updatedSteps[insertIndex].error = undefined;
|
|
876
|
-
|
|
877
|
-
// Report inserted step
|
|
878
|
-
if (this.progressReporter?.onStepProgress && jobId) {
|
|
879
|
-
await this.progressReporter.onStepProgress({
|
|
880
|
-
jobId,
|
|
881
|
-
stepId: updatedSteps[insertIndex].id, // Preserve original step ID if provided
|
|
882
|
-
stepNumber: insertIndex + 1,
|
|
883
|
-
description: updatedSteps[insertIndex].description,
|
|
884
|
-
code: updatedSteps[insertIndex].code,
|
|
885
|
-
status: 'SUCCESS_STEP_EXECUTION' as any,
|
|
886
|
-
wasRepaired: true
|
|
887
|
-
});
|
|
888
|
-
}
|
|
889
|
-
}
|
|
890
|
-
} else if (repairAction.operation === StepOperation.REMOVE) {
|
|
891
|
-
// For REMOVE: no step to mark as successful since we removed it
|
|
892
|
-
// The step is already removed from the array
|
|
893
|
-
}
|
|
894
|
-
|
|
895
|
-
const commandInfo = repairAction.operation === StepOperation.MODIFY ?
|
|
896
|
-
`MODIFY: "${repairAction.newStep?.code || 'N/A'}"` :
|
|
897
|
-
repairAction.operation === StepOperation.INSERT ?
|
|
898
|
-
`INSERT: "${repairAction.newStep?.code || 'N/A'}"` :
|
|
899
|
-
repairAction.operation === StepOperation.REMOVE ?
|
|
900
|
-
`REMOVE: step at index ${repairAction.stepIndex}` :
|
|
901
|
-
repairAction.operation;
|
|
902
|
-
this.log(`Step ${i + 1} repair action ${commandInfo} executed successfully on attempt ${attempt}${usedVisionMode ? ' (vision-aided)' : ''}`);
|
|
903
|
-
|
|
904
|
-
// Update execution context based on the repair action
|
|
905
|
-
if (repairAction.operation === StepOperation.MODIFY && repairAction.newStep) {
|
|
906
|
-
// Update the step in the execution context for variable tracking
|
|
907
|
-
executionContext = executionContext.replace(originalCode, repairAction.newStep.code);
|
|
908
|
-
} else if (repairAction.operation === StepOperation.INSERT && repairAction.newStep) {
|
|
909
|
-
// Insert the new step code into execution context for variable tracking
|
|
910
|
-
executionContext += repairAction.newStep.code + '\n';
|
|
911
|
-
} else if (repairAction.operation === StepOperation.REMOVE) {
|
|
912
|
-
// Remove the step code from execution context for variable tracking
|
|
913
|
-
executionContext = executionContext.replace(originalCode, '');
|
|
914
|
-
}
|
|
915
|
-
|
|
916
|
-
// Record this successful repair
|
|
917
|
-
recentRepairs.push({
|
|
918
|
-
stepNumber: i + 1,
|
|
919
|
-
operation: repairAction.operation,
|
|
920
|
-
originalDescription: repairAction.operation === StepOperation.REMOVE ? originalDescription : undefined,
|
|
921
|
-
newDescription: repairAction.newStep?.description,
|
|
922
|
-
originalCode: repairAction.operation === StepOperation.REMOVE ? originalCode : undefined,
|
|
923
|
-
newCode: repairAction.newStep?.code
|
|
924
|
-
});
|
|
925
|
-
|
|
926
|
-
// Keep only the last 3 repairs for context
|
|
927
|
-
if (recentRepairs.length > 3) {
|
|
928
|
-
recentRepairs.shift();
|
|
929
|
-
}
|
|
930
|
-
|
|
931
|
-
// Update step index based on operation
|
|
932
|
-
if (repairAction.operation === StepOperation.INSERT) {
|
|
933
|
-
// For INSERT: inserted step is already executed
|
|
934
|
-
this.log(`INSERT operation: current i=${i}, insertAfterIndex=${repairAction.insertAfterIndex}`);
|
|
935
|
-
this.log(`INSERT: Steps array length before: ${updatedSteps.length}`);
|
|
936
|
-
this.log(`INSERT: Steps before operation: ${updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`).join(', ')}`);
|
|
937
|
-
|
|
938
|
-
if (repairAction.insertAfterIndex !== undefined && repairAction.insertAfterIndex < i) {
|
|
939
|
-
// If inserting before current position, current step moved down by 1
|
|
940
|
-
this.log(`INSERT before current position: incrementing i from ${i} to ${i + 1}`);
|
|
941
|
-
i++; // Move to the original step that was pushed to the next position
|
|
942
|
-
} else {
|
|
943
|
-
// If inserting at or after current position, stay at current step
|
|
944
|
-
this.log(`INSERT at/after current position: keeping i at ${i}`);
|
|
945
|
-
}
|
|
946
|
-
|
|
947
|
-
this.log(`INSERT: Steps array length after: ${updatedSteps.length}`);
|
|
948
|
-
this.log(`INSERT: Steps after operation: ${updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`).join(', ')}`);
|
|
949
|
-
} else if (repairAction.operation === StepOperation.REMOVE) {
|
|
950
|
-
// For REMOVE: stay at same index since the next step moved to current position
|
|
951
|
-
// Don't increment i because the array shifted left
|
|
952
|
-
} else {
|
|
953
|
-
// For MODIFY: move to next step since modified step was executed
|
|
954
|
-
i++; // Move to next step for MODIFY
|
|
955
|
-
}
|
|
956
|
-
|
|
957
|
-
// Add the repaired step's code to execution context for variable tracking
|
|
958
|
-
executionContext += step.code + '\n';
|
|
959
|
-
|
|
960
|
-
break;
|
|
961
|
-
} else {
|
|
962
|
-
throw new Error(result.error || 'Repair action failed');
|
|
963
|
-
}
|
|
964
|
-
} catch (repairError) {
|
|
965
|
-
const repairErrorMessage = repairError instanceof Error ? repairError.message : 'Repair failed';
|
|
966
|
-
const commandInfo = repairSuggestion.action.operation === StepOperation.MODIFY ?
|
|
967
|
-
`MODIFY: "${repairSuggestion.action.newStep?.code || 'N/A'}"` :
|
|
968
|
-
repairSuggestion.action.operation === StepOperation.INSERT ?
|
|
969
|
-
`INSERT: "${repairSuggestion.action.newStep?.code || 'N/A'}"` :
|
|
970
|
-
repairSuggestion.action.operation === StepOperation.REMOVE ?
|
|
971
|
-
`REMOVE: step at index ${repairSuggestion.action.stepIndex}` :
|
|
972
|
-
repairSuggestion.action.operation;
|
|
973
|
-
this.log(`Step ${i + 1} repair attempt ${attempt} failed (${commandInfo}): ${repairErrorMessage}`);
|
|
974
|
-
if (repairError instanceof Error && repairError.stack) {
|
|
975
|
-
this.log(` Repair stack trace: ${repairError.stack}`);
|
|
976
|
-
}
|
|
977
|
-
|
|
978
|
-
// Record this attempt in history
|
|
979
|
-
repairHistory.push({
|
|
980
|
-
attempt,
|
|
981
|
-
action: repairSuggestion.action,
|
|
982
|
-
error: repairErrorMessage,
|
|
983
|
-
pageInfo
|
|
984
|
-
});
|
|
985
|
-
|
|
986
|
-
step.error = repairErrorMessage;
|
|
987
|
-
}
|
|
988
|
-
}
|
|
989
|
-
|
|
990
|
-
if (!repairSuccess) {
|
|
991
|
-
this.log(`Step ${i + 1} failed after ${maxTries} repair attempts`);
|
|
992
|
-
break;
|
|
993
|
-
}
|
|
994
|
-
}
|
|
995
|
-
}
|
|
996
|
-
|
|
997
|
-
return updatedSteps;
|
|
998
|
-
}
|
|
999
|
-
|
|
1000
|
-
private async executeStepCode(code: string, page: any): Promise<void> {
|
|
1001
|
-
// Keep default timeout (5 seconds) for fast feedback on wrong selectors
|
|
1002
|
-
// Navigation operations should use explicit longer timeouts in generated code
|
|
1003
|
-
page.setDefaultTimeout(5000);
|
|
1004
|
-
|
|
1005
|
-
try {
|
|
1006
|
-
// Clean and validate the code before execution
|
|
1007
|
-
const cleanedCode = this.cleanStepCode(code);
|
|
1008
|
-
|
|
1009
|
-
if (!cleanedCode || cleanedCode.trim().length === 0) {
|
|
1010
|
-
throw new Error('Step code is empty or contains only comments');
|
|
1011
|
-
}
|
|
1012
|
-
|
|
1013
|
-
// Dynamically import expect
|
|
1014
|
-
const { expect } = require('@playwright/test');
|
|
1015
|
-
|
|
1016
|
-
// Create an async function that has access to page, expect, and other Playwright globals
|
|
1017
|
-
const executeCode = new Function('page', 'expect', `return (async () => { ${cleanedCode} })()`);
|
|
1018
|
-
const result = executeCode(page, expect);
|
|
1019
|
-
await result;
|
|
1020
|
-
} finally {
|
|
1021
|
-
// Ensure timeout remains consistent
|
|
1022
|
-
page.setDefaultTimeout(5000);
|
|
1023
|
-
}
|
|
1024
|
-
}
|
|
1025
|
-
|
|
1026
|
-
/**
|
|
1027
|
-
* Validate step code has executable content (preserves comments)
|
|
1028
|
-
*/
|
|
1029
|
-
private cleanStepCode(code: string): string {
|
|
1030
|
-
if (!code || code.trim().length === 0) {
|
|
1031
|
-
return '';
|
|
1032
|
-
}
|
|
1033
|
-
|
|
1034
|
-
// Check if there are any executable statements (including those with comments)
|
|
1035
|
-
const hasExecutableCode = /[a-zA-Z_$][a-zA-Z0-9_$]*\s*\(|await\s+|return\s+|if\s*\(|for\s*\(|while\s*\(|switch\s*\(|try\s*\{|catch\s*\(/.test(code);
|
|
1036
|
-
|
|
1037
|
-
if (!hasExecutableCode) {
|
|
1038
|
-
return '';
|
|
1039
|
-
}
|
|
1040
|
-
|
|
1041
|
-
return code; // Return the original code without removing comments
|
|
1042
|
-
}
|
|
1043
|
-
|
|
1044
|
-
private async executeStepInContext(
|
|
1045
|
-
code: string,
|
|
1046
|
-
page: any,
|
|
1047
|
-
executionContext: string,
|
|
1048
|
-
contextVariables: Map<string, any>
|
|
1049
|
-
): Promise<void> {
|
|
1050
|
-
// Detect if code contains navigation or load state operations that need longer timeout
|
|
1051
|
-
const needsLongerTimeout = code.includes('waitForLoadState') ||
|
|
1052
|
-
code.includes('goto(') ||
|
|
1053
|
-
code.includes('waitForURL') ||
|
|
1054
|
-
code.includes('waitForNavigation');
|
|
1055
|
-
|
|
1056
|
-
// Use appropriate timeout based on operation type
|
|
1057
|
-
const timeout = needsLongerTimeout ? 30000 : 5000;
|
|
1058
|
-
page.setDefaultTimeout(timeout);
|
|
1059
|
-
|
|
1060
|
-
try {
|
|
1061
|
-
// Execute only the current step code, but make context variables available
|
|
1062
|
-
const fullCode = code;
|
|
1063
|
-
|
|
1064
|
-
// Dynamically import expect
|
|
1065
|
-
const { expect } = require('@playwright/test');
|
|
1066
|
-
|
|
1067
|
-
// Create a function that has access to page, expect, and the context variables
|
|
1068
|
-
const executeCode = new Function(
|
|
1069
|
-
'page',
|
|
1070
|
-
'expect',
|
|
1071
|
-
'contextVariables',
|
|
1072
|
-
`return (async () => {
|
|
1073
|
-
// Make context variables available in the execution scope
|
|
1074
|
-
for (const [key, value] of contextVariables) {
|
|
1075
|
-
globalThis[key] = value;
|
|
1076
|
-
}
|
|
1077
|
-
|
|
1078
|
-
${fullCode}
|
|
1079
|
-
|
|
1080
|
-
// Capture any new variables that might have been created
|
|
1081
|
-
const newVars = {};
|
|
1082
|
-
for (const key in globalThis) {
|
|
1083
|
-
if (!contextVariables.has(key) && typeof globalThis[key] !== 'function' && key !== 'page' && key !== 'expect') {
|
|
1084
|
-
newVars[key] = globalThis[key];
|
|
1085
|
-
}
|
|
1086
|
-
}
|
|
1087
|
-
return newVars;
|
|
1088
|
-
})()`
|
|
1089
|
-
);
|
|
1090
|
-
|
|
1091
|
-
const newVars = await executeCode(page, expect, contextVariables);
|
|
1092
|
-
|
|
1093
|
-
// Update the context variables with any new variables created
|
|
1094
|
-
for (const [key, value] of Object.entries(newVars)) {
|
|
1095
|
-
contextVariables.set(key, value);
|
|
1096
|
-
}
|
|
1097
|
-
} finally {
|
|
1098
|
-
// Reset to default timeout for element operations
|
|
1099
|
-
page.setDefaultTimeout(5000);
|
|
1100
|
-
}
|
|
1101
|
-
}
|
|
1102
|
-
|
|
1103
|
-
private async executeScriptContent(script: string, page: any): Promise<void> {
|
|
1104
|
-
// Extract the test function content
|
|
1105
|
-
const testMatch = script.match(/test\([^,]+,\s*async\s*\(\s*\{\s*page[^}]*\}\s*\)\s*=>\s*\{([\s\S]*)\}\s*\);/);
|
|
1106
|
-
if (!testMatch) {
|
|
1107
|
-
throw new Error('Could not extract test function from script');
|
|
1108
|
-
}
|
|
1109
|
-
|
|
1110
|
-
const testBody = testMatch[1];
|
|
1111
|
-
|
|
1112
|
-
// Dynamically import expect
|
|
1113
|
-
const { expect } = require('@playwright/test');
|
|
1114
|
-
|
|
1115
|
-
// Execute the entire test body as one async function
|
|
1116
|
-
const executeTest = new Function('page', 'expect', `return (async () => { ${testBody} })()`);
|
|
1117
|
-
await executeTest(page, expect);
|
|
1118
|
-
}
|
|
1119
|
-
|
|
1120
|
-
private async getEnhancedPageInfo(page: any): Promise<PageInfo> {
|
|
1121
|
-
try {
|
|
1122
|
-
return await getEnhancedPageInfo(page);
|
|
1123
|
-
} catch (error) {
|
|
1124
|
-
return {
|
|
1125
|
-
url: page.url(),
|
|
1126
|
-
title: 'Unknown',
|
|
1127
|
-
ariaSnapshot: null,
|
|
1128
|
-
interactiveElements: [],
|
|
1129
|
-
formattedElements: 'Unable to extract'
|
|
1130
|
-
};
|
|
1131
|
-
}
|
|
1132
|
-
}
|
|
1133
|
-
|
|
1134
|
-
private buildFailureHistory(
|
|
1135
|
-
repairHistory: Array<{ attempt: number; action: StepRepairAction; error: string; pageInfo: PageInfo }>,
|
|
1136
|
-
originalStep: ScriptStep,
|
|
1137
|
-
originalError: any
|
|
1138
|
-
): string {
|
|
1139
|
-
if (repairHistory.length === 0) {
|
|
1140
|
-
return `Original failure: ${this.safeSerializeError(originalError)}`;
|
|
1141
|
-
}
|
|
1142
|
-
|
|
1143
|
-
let history = `Original failure: ${this.safeSerializeError(originalError)}\n\n`;
|
|
1144
|
-
history += `Previous repair attempts:\n`;
|
|
1145
|
-
|
|
1146
|
-
repairHistory.forEach((attempt, index) => {
|
|
1147
|
-
history += `Attempt ${attempt.attempt}:\n`;
|
|
1148
|
-
history += ` Operation: ${attempt.action.operation}\n`;
|
|
1149
|
-
if (attempt.action.newStep) {
|
|
1150
|
-
history += ` Description: ${attempt.action.newStep.description}\n`;
|
|
1151
|
-
history += ` Code: ${attempt.action.newStep.code}\n`;
|
|
1152
|
-
}
|
|
1153
|
-
history += ` Error: ${attempt.error}\n`;
|
|
1154
|
-
if (index < repairHistory.length - 1) {
|
|
1155
|
-
history += `\n`;
|
|
1156
|
-
}
|
|
1157
|
-
});
|
|
1158
|
-
|
|
1159
|
-
return history;
|
|
1160
|
-
}
|
|
1161
|
-
|
|
1162
|
-
private buildRecentRepairsContext(
|
|
1163
|
-
recentRepairs: Array<{
|
|
1164
|
-
stepNumber: number;
|
|
1165
|
-
operation: string;
|
|
1166
|
-
originalDescription?: string;
|
|
1167
|
-
newDescription?: string;
|
|
1168
|
-
originalCode?: string;
|
|
1169
|
-
newCode?: string;
|
|
1170
|
-
}>
|
|
1171
|
-
): string {
|
|
1172
|
-
if (recentRepairs.length === 0) {
|
|
1173
|
-
return 'No recent repairs to consider.';
|
|
1174
|
-
}
|
|
1175
|
-
|
|
1176
|
-
let context = 'Recent successful repairs that may affect this step:\n\n';
|
|
1177
|
-
|
|
1178
|
-
recentRepairs.forEach((repair, index) => {
|
|
1179
|
-
context += `Step ${repair.stepNumber} was successfully repaired:\n`;
|
|
1180
|
-
context += ` Operation: ${repair.operation}\n`;
|
|
1181
|
-
|
|
1182
|
-
if (repair.operation === 'REMOVE') {
|
|
1183
|
-
context += ` Removed: "${repair.originalDescription}"\n`;
|
|
1184
|
-
context += ` Code removed:\n ${repair.originalCode?.replace(/\n/g, '\n ')}\n`;
|
|
1185
|
-
} else if (repair.operation === 'INSERT') {
|
|
1186
|
-
context += ` Inserted: "${repair.newDescription}"\n`;
|
|
1187
|
-
context += ` Code inserted:\n ${repair.newCode?.replace(/\n/g, '\n ')}\n`;
|
|
1188
|
-
} else {
|
|
1189
|
-
context += ` Original: "${repair.originalDescription}"\n`;
|
|
1190
|
-
context += ` Repaired: "${repair.newDescription}"\n`;
|
|
1191
|
-
context += ` Code changed from:\n ${repair.originalCode?.replace(/\n/g, '\n ')}\n`;
|
|
1192
|
-
context += ` To:\n ${repair.newCode?.replace(/\n/g, '\n ')}\n`;
|
|
1193
|
-
}
|
|
1194
|
-
|
|
1195
|
-
if (index < recentRepairs.length - 1) {
|
|
1196
|
-
context += `\n`;
|
|
1197
|
-
}
|
|
1198
|
-
});
|
|
1199
|
-
|
|
1200
|
-
context += '\nConsider how these changes might affect the current step and adjust accordingly.';
|
|
1201
|
-
return context;
|
|
1202
|
-
}
|
|
1203
|
-
|
|
1204
|
-
private async applyRepairActionInContext(
|
|
1205
|
-
action: StepRepairAction,
|
|
1206
|
-
steps: (ScriptStep & { success?: boolean; error?: string })[],
|
|
1207
|
-
currentIndex: number,
|
|
1208
|
-
page: any,
|
|
1209
|
-
executionContext: string,
|
|
1210
|
-
contextVariables: Map<string, any>
|
|
1211
|
-
): Promise<{ success: boolean; error?: string; updatedContext?: string }> {
|
|
1212
|
-
try {
|
|
1213
|
-
switch (action.operation) {
|
|
1214
|
-
case StepOperation.MODIFY:
|
|
1215
|
-
if (action.newStep && action.stepIndex !== undefined) {
|
|
1216
|
-
// Modify existing step
|
|
1217
|
-
steps[action.stepIndex] = {
|
|
1218
|
-
...action.newStep,
|
|
1219
|
-
success: false,
|
|
1220
|
-
error: undefined
|
|
1221
|
-
};
|
|
1222
|
-
// Test the modified step with current page state and variables
|
|
1223
|
-
await this.executeStepCode(action.newStep.code, page);
|
|
1224
|
-
return { success: true, updatedContext: executionContext + action.newStep.code };
|
|
1225
|
-
}
|
|
1226
|
-
break;
|
|
1227
|
-
|
|
1228
|
-
case StepOperation.INSERT:
|
|
1229
|
-
if (action.newStep && action.insertAfterIndex !== undefined) {
|
|
1230
|
-
// Insert new step after specified index
|
|
1231
|
-
const insertIndex = action.insertAfterIndex + 1;
|
|
1232
|
-
const newStep = {
|
|
1233
|
-
...action.newStep,
|
|
1234
|
-
success: false,
|
|
1235
|
-
error: undefined
|
|
1236
|
-
};
|
|
1237
|
-
this.log(`INSERT: Inserting step at index ${insertIndex} with description "${newStep.description}"`);
|
|
1238
|
-
this.log(`INSERT: Steps before insertion: ${steps.map((s, i) => `${i}: "${s.description}" (success: ${s.success})`).join(', ')}`);
|
|
1239
|
-
|
|
1240
|
-
// Preserve success status of existing steps before insertion
|
|
1241
|
-
const successStatusMap = new Map(steps.map((step, index) => [index, { success: step.success, error: step.error }]));
|
|
1242
|
-
|
|
1243
|
-
steps.splice(insertIndex, 0, newStep);
|
|
1244
|
-
|
|
1245
|
-
// Restore success status for steps that were shifted by the insertion
|
|
1246
|
-
// Steps at insertIndex and before keep their original status
|
|
1247
|
-
// Steps after insertIndex need to be shifted to their new positions
|
|
1248
|
-
for (let i = insertIndex + 1; i < steps.length; i++) {
|
|
1249
|
-
const originalIndex = i - 1; // The step that was originally at this position
|
|
1250
|
-
if (successStatusMap.has(originalIndex)) {
|
|
1251
|
-
const status = successStatusMap.get(originalIndex)!;
|
|
1252
|
-
steps[i].success = status.success;
|
|
1253
|
-
steps[i].error = status.error;
|
|
1254
|
-
}
|
|
1255
|
-
}
|
|
1256
|
-
|
|
1257
|
-
// CRITICAL FIX: Ensure the inserted step doesn't overwrite existing step data
|
|
1258
|
-
// The new step should only have its own description, not inherit from existing steps
|
|
1259
|
-
this.log(`INSERT: Final step array after restoration: ${steps.map((s, i) => `${i}: "${s.description}" (success: ${s.success})`).join(', ')}`);
|
|
1260
|
-
|
|
1261
|
-
this.log(`INSERT: Steps after insertion: ${steps.map((s, i) => `${i}: "${s.description}" (success: ${s.success})`).join(', ')}`);
|
|
1262
|
-
// Test the new step with current page state
|
|
1263
|
-
await this.executeStepCode(action.newStep.code, page);
|
|
1264
|
-
return { success: true, updatedContext: executionContext + action.newStep.code };
|
|
1265
|
-
}
|
|
1266
|
-
break;
|
|
1267
|
-
|
|
1268
|
-
case StepOperation.REMOVE:
|
|
1269
|
-
if (action.stepIndex !== undefined) {
|
|
1270
|
-
// Remove step
|
|
1271
|
-
steps.splice(action.stepIndex, 1);
|
|
1272
|
-
return { success: true, updatedContext: executionContext };
|
|
1273
|
-
}
|
|
1274
|
-
break;
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
return { success: false, error: 'Invalid repair action' };
|
|
1278
|
-
} catch (error) {
|
|
1279
|
-
return {
|
|
1280
|
-
success: false,
|
|
1281
|
-
error: error instanceof Error ? error.message : 'Unknown error during repair action'
|
|
1282
|
-
};
|
|
1283
|
-
}
|
|
1284
|
-
}
|
|
1285
|
-
|
|
1286
|
-
private async applyRepairAction(
|
|
1287
|
-
action: StepRepairAction,
|
|
1288
|
-
steps: (ScriptStep & { success?: boolean; error?: string })[],
|
|
1289
|
-
currentIndex: number,
|
|
1290
|
-
page: any
|
|
1291
|
-
): Promise<{ success: boolean; error?: string }> {
|
|
1292
|
-
try {
|
|
1293
|
-
switch (action.operation) {
|
|
1294
|
-
case StepOperation.MODIFY:
|
|
1295
|
-
if (action.newStep && action.stepIndex !== undefined) {
|
|
1296
|
-
// Modify existing step
|
|
1297
|
-
steps[action.stepIndex] = {
|
|
1298
|
-
...action.newStep,
|
|
1299
|
-
success: false,
|
|
1300
|
-
error: undefined
|
|
1301
|
-
};
|
|
1302
|
-
// Test the modified step
|
|
1303
|
-
await this.executeStepCode(action.newStep.code, page);
|
|
1304
|
-
return { success: true };
|
|
1305
|
-
}
|
|
1306
|
-
break;
|
|
1307
|
-
|
|
1308
|
-
case StepOperation.INSERT:
|
|
1309
|
-
if (action.newStep && action.insertAfterIndex !== undefined) {
|
|
1310
|
-
// Insert new step after specified index
|
|
1311
|
-
const insertIndex = action.insertAfterIndex + 1;
|
|
1312
|
-
const newStep = {
|
|
1313
|
-
...action.newStep,
|
|
1314
|
-
success: false,
|
|
1315
|
-
error: undefined
|
|
1316
|
-
};
|
|
1317
|
-
steps.splice(insertIndex, 0, newStep);
|
|
1318
|
-
// Test the inserted step
|
|
1319
|
-
await this.executeStepCode(action.newStep.code, page);
|
|
1320
|
-
return { success: true };
|
|
1321
|
-
}
|
|
1322
|
-
break;
|
|
1323
|
-
|
|
1324
|
-
case StepOperation.REMOVE:
|
|
1325
|
-
if (action.stepIndex !== undefined) {
|
|
1326
|
-
// Remove the step
|
|
1327
|
-
steps.splice(action.stepIndex, 1);
|
|
1328
|
-
return { success: true };
|
|
1329
|
-
}
|
|
1330
|
-
break;
|
|
1331
|
-
}
|
|
1332
|
-
|
|
1333
|
-
return { success: false, error: 'Invalid repair action' };
|
|
1334
|
-
} catch (error) {
|
|
1335
|
-
return {
|
|
1336
|
-
success: false,
|
|
1337
|
-
error: error instanceof Error ? error.message : 'Repair action execution failed'
|
|
1338
|
-
};
|
|
1339
|
-
}
|
|
1340
|
-
}
|
|
1341
|
-
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
private generateUpdatedScript(steps: (ScriptStep & { success?: boolean; error?: string })[], repairAdvice?: string): string {
|
|
1345
|
-
const scriptLines = [
|
|
1346
|
-
"import { test, expect } from '@playwright/test';",
|
|
1347
|
-
`test('repairedTest', async ({ page, browser, context }) => {`
|
|
1348
|
-
];
|
|
1349
|
-
|
|
1350
|
-
steps.forEach((step, index) => {
|
|
1351
|
-
// Only add step if it has code to execute
|
|
1352
|
-
if (step.code && step.code.trim().length > 0) {
|
|
1353
|
-
scriptLines.push(` // ${step.description}`);
|
|
1354
|
-
const codeLines = step.code.split('\n');
|
|
1355
|
-
codeLines.forEach(line => {
|
|
1356
|
-
scriptLines.push(` ${line}`);
|
|
1357
|
-
});
|
|
1358
|
-
}
|
|
1359
|
-
});
|
|
1360
|
-
|
|
1361
|
-
scriptLines.push('});');
|
|
1362
|
-
const script = scriptLines.join('\n');
|
|
1363
|
-
|
|
1364
|
-
// Add TestChimp comment to the repaired script with repair advice
|
|
1365
|
-
return addTestChimpComment(script, repairAdvice);
|
|
1366
|
-
}
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
/**
|
|
1370
|
-
* Initialize browser with configuration (delegates to utility function)
|
|
1371
|
-
*/
|
|
1372
|
-
private async initializeBrowser(playwrightConfig?: string, headless?: boolean, playwrightConfigFilePath?: string): Promise<{ browser: any; context: any; page: any }> {
|
|
1373
|
-
return initializeBrowser(playwrightConfig, headless, playwrightConfigFilePath, this.logger);
|
|
1374
|
-
}
|
|
1375
|
-
|
|
1376
|
-
/**
|
|
1377
|
-
* Safely serialize error information, filtering out non-serializable values
|
|
1378
|
-
*/
|
|
1379
|
-
private safeSerializeError(error: any): string {
|
|
1380
|
-
try {
|
|
1381
|
-
if (error instanceof Error) {
|
|
1382
|
-
return error.message;
|
|
1383
|
-
}
|
|
1384
|
-
|
|
1385
|
-
if (typeof error === 'string') {
|
|
1386
|
-
return error;
|
|
1387
|
-
}
|
|
1388
|
-
|
|
1389
|
-
if (typeof error === 'object' && error !== null) {
|
|
1390
|
-
// Try to extract meaningful information without serializing the entire object
|
|
1391
|
-
const safeError: any = {};
|
|
1392
|
-
|
|
1393
|
-
// Copy safe properties
|
|
1394
|
-
if (error.message) safeError.message = error.message;
|
|
1395
|
-
if (error.name) safeError.name = error.name;
|
|
1396
|
-
if (error.code) safeError.code = error.code;
|
|
1397
|
-
if (error.status) safeError.status = error.status;
|
|
1398
|
-
|
|
1399
|
-
// Try to get stack trace safely
|
|
1400
|
-
if (error.stack && typeof error.stack === 'string') {
|
|
1401
|
-
safeError.stack = error.stack;
|
|
1402
|
-
}
|
|
1403
|
-
|
|
1404
|
-
return JSON.stringify(safeError);
|
|
1405
|
-
}
|
|
1406
|
-
|
|
1407
|
-
return String(error);
|
|
1408
|
-
} catch (serializationError) {
|
|
1409
|
-
// If even safe serialization fails, return a basic string representation
|
|
1410
|
-
return `Error: ${String(error)}`;
|
|
1411
|
-
}
|
|
1412
|
-
}
|
|
1413
|
-
}
|