testchimp-runner-core 0.0.34 → 0.0.36

package/dist/orchestrator/orchestrator-agent.js

@@ -5,13 +5,16 @@
  */
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.OrchestratorAgent = void 0;
-const test_1 = require("@playwright/test");
 const progress_reporter_1 = require("../progress-reporter");
 const page_info_utils_1 = require("../utils/page-info-utils");
 const coordinate_converter_1 = require("../utils/coordinate-converter");
 const model_constants_1 = require("../model-constants");
 const types_1 = require("./types");
 const orchestrator_prompts_1 = require("./orchestrator-prompts");
+const page_info_retry_1 = require("../utils/page-info-retry");
+const decision_parser_1 = require("./decision-parser");
+const page_som_handler_1 = require("./page-som-handler");
+const som_types_1 = require("./som-types");
 /**
  * Orchestrator Agent - manages step execution with tool use and memory
  */
@@ -24,6 +27,11 @@ class OrchestratorAgent {
         this.progressReporter = progressReporter;
         this.logger = logger;
         this.debugMode = debugMode || false;
+        this.decisionParser = new decision_parser_1.DecisionParser(logger);
+        // Initialize SoM handler if enabled
+        if (this.config.useSoM) {
+            this.somHandler = new page_som_handler_1.PageSoMHandler(null, this.logger);
+        }
     }
     setDebugMode(enabled) {
         this.debugMode = enabled;
@@ -31,11 +39,12 @@ class OrchestratorAgent {
     /**
      * Execute a single step of the scenario
      */
-    async executeStep(page, stepDescription, stepNumber, totalSteps, scenarioSteps, memory, jobId) {
+    async executeStep(page, stepDescription, stepNumber, totalSteps, scenarioSteps, memory, jobId, priorSteps, // NEW: For repair mode (undefined for script gen)
+    nextSteps // NEW: For repair mode (undefined for script gen)
+    ) {
         this.logger?.(`\n[Orchestrator] ========== STEP ${stepNumber}/${totalSteps} ==========`);
         this.logger?.(`[Orchestrator] 🎯 Goal: ${stepDescription}`);
         let iteration = 0;
-        let previousReflection = undefined;
         let noteToSelf = memory.latestNote; // Start with note from previous step
         const commandsExecuted = [];
         let consecutiveFailures = 0; // Track consecutive iterations with failed commands
@@ -44,24 +53,45 @@ class OrchestratorAgent {
             iteration++;
             this.logger?.(`\n[Orchestrator] === Iteration ${iteration}/${this.config.maxIterationsPerStep} ===`);
             // Build context for agent
-            const context = await this.buildAgentContext(page, stepDescription, stepNumber, totalSteps, scenarioSteps, memory, previousReflection, consecutiveFailures, noteToSelf // NEW: Pass note from previous iteration
+            const context = await this.buildAgentContext(page, stepDescription, stepNumber, totalSteps, scenarioSteps, memory, consecutiveFailures, noteToSelf, // Pass note from previous iteration
+            priorSteps, // NEW: Pass repair context
+            nextSteps // NEW: Pass repair context
             );
             // Call agent to make decision
             const decision = await this.callAgent(context, jobId, stepNumber, iteration, consecutiveFailures);
             // Log agent's reasoning
-            this.logAgentDecision(decision, iteration);
+            this.decisionParser.log(decision, iteration);
             // Report progress
             await this.reportStepProgress(jobId, stepNumber, stepDescription, decision, iteration);
             // Execute tools if requested (tools are READ-ONLY, they don't change state)
             let toolResults = {};
-            // ANTI-LOOP: Detect if agent is taking screenshots repeatedly without acting
+            // ANTI-LOOP: Detect and BLOCK screenshot loops (PER STEP)
+            const screenshotsThisStep = memory.history.filter(s => s.stepNumber === stepNumber &&
+                (s.code.includes('take_screenshot') || s.action.includes('Screenshot')));
             const recentScreenshots = memory.history.slice(-3).filter(s => s.code.includes('take_screenshot') || s.action.includes('Screenshot'));
-            if (recentScreenshots.length >= 2 && iteration >= 3) {
+            if (screenshotsThisStep.length >= 3) {
+                this.logger?.(`[Orchestrator] 🚨 SCREENSHOT LOOP - ${screenshotsThisStep.length} screenshots THIS STEP! BLOCKING further screenshots`, 'error');
+            }
+            else if (recentScreenshots.length >= 2 && iteration >= 3) {
                 this.logger?.(`[Orchestrator] ⚠️  WARNING: ${recentScreenshots.length} screenshots in last 3 iterations - agent may be looping`, 'warn');
-                this.logger?.(`[Orchestrator] 💭 System: Stop gathering info, START ACTING with available selectors`);
             }
             if (decision.toolCalls && decision.toolCalls.length > 0) {
-                toolResults = await this.executeTools(decision.toolCalls, page, memory, stepNumber);
+                // ENFORCE: Block screenshot tool calls if too many taken IN THIS STEP
+                if (screenshotsThisStep.length >= 3) {
+                    decision.toolCalls = decision.toolCalls.filter(tc => tc.name !== 'take_screenshot');
+                    if (decision.toolCalls.length === 0) {
+                        this.logger?.(`[Orchestrator] 🚫 REJECTED screenshot tool call - loop detected. Agent must ACT.`, 'warn');
+                        toolResults = [{
+                                toolName: 'take_screenshot',
+                                success: false,
+                                error: 'SYSTEM BLOCKED: Too many screenshots taken. You must use existing DOM snapshots and execute commands now. Analysis paralysis detected.',
+                                data: null
+                            }];
+                    }
+                }
+                if (decision.toolCalls.length > 0) {
+                    toolResults = await this.executeTools(decision.toolCalls, page, memory, stepNumber, context.currentPageInfo.refMap);
+                }
                 // If agent wants to wait for tool results before proceeding, call agent again
                 if (decision.needsToolResults) {
                     const updatedContext = { ...context, toolResults };
@@ -80,7 +110,7 @@ class OrchestratorAgent {
             if (decision.blockerDetected && decision.blockerDetected.clearingCommands && decision.blockerDetected.clearingCommands.length > 0) {
                 this.logger?.(`[Orchestrator] 🚧 BLOCKER DETECTED: ${decision.blockerDetected.description}`);
                 this.logger?.(`[Orchestrator] 🧹 Clearing blocker with ${decision.blockerDetected.clearingCommands.length} command(s)...`);
-                const blockerResult = await this.executeCommandsSequentially(decision.blockerDetected.clearingCommands, page, memory, stepNumber, iteration, jobId);
+                const blockerResult = await this.executeCommands(decision.blockerDetected.clearingCommands, page, memory, stepNumber, iteration, jobId);
                 // Add blocker commands with comment to output
                 if (blockerResult.executed.length > 0) {
                     commandsExecuted.push(`// Blocker: ${decision.blockerDetected.description}`);
@@ -94,8 +124,8 @@ class OrchestratorAgent {
                 }
             }
             // Execute main commands (only if no blocker failure)
-            if (decision.commands && decision.commands.length > 0 && !iterationHadFailure) {
-                const executeResult = await this.executeCommandsSequentially(decision.commands, page, memory, stepNumber, iteration, jobId);
+            if (!iterationHadFailure && decision.commands && decision.commands.length > 0) {
+                const executeResult = await this.executeCommands(decision.commands, page, memory, stepNumber, iteration, jobId);
                 commandsExecuted.push(...executeResult.executed);
                 // Track failures
                 if (!executeResult.allSucceeded) {
@@ -107,8 +137,8 @@ class OrchestratorAgent {
                     consecutiveFailures = 0; // Reset on success
                 }
             }
-            // Handle coordinate-based actions (NEW - fallback when selectors fail)
-            if (decision.coordinateAction && !iterationHadFailure) {
+            // Handle coordinate-based actions (NEW - fallback when selectors fail) - ONLY if enabled
+            if (this.config.enableCoordinateMode && decision.coordinateAction && !iterationHadFailure) {
                 coordinateAttempts++;
                 this.logger?.(`[Orchestrator] 🎯 Coordinate Action (attempt ${coordinateAttempts}/2): ${decision.coordinateAction.action} at (${decision.coordinateAction.xPercent}%, ${decision.coordinateAction.yPercent}%)`);
                 try {
@@ -120,7 +150,7 @@ class OrchestratorAgent {
                     this.logger?.(`[Orchestrator] Generated commands:`);
                     coordCommands.forEach(cmd => this.logger?.(`  ${cmd}`));
                     // Execute coordinate commands
-                    const coordResult = await this.executeCommandsSequentially(coordCommands, page, memory, stepNumber, iteration, jobId);
+                    const coordResult = await this.executeCommands(coordCommands, page, memory, stepNumber, iteration, jobId);
                     commandsExecuted.push(...coordResult.executed);
                     if (!coordResult.allSucceeded) {
                         this.logger?.(`[Orchestrator] ❌ Coordinate action failed (Playwright error)`);
@@ -263,9 +293,7 @@ class OrchestratorAgent {
                     memory.experiences = memory.experiences.slice(-this.config.maxExperiences);
                 }
             }
-            // Store self-reflection for next iteration
-            previousReflection = decision.selfReflection;
-            // Store note to future self (NEW - tactical memory across iterations AND steps)
+            // Store note to future self (tactical memory across iterations AND steps)
             if (decision.noteToFutureSelf) {
                 noteToSelf = {
                     fromIteration: iteration,
@@ -312,15 +340,46 @@ class OrchestratorAgent {
     /**
      * Build context for agent
      */
-    async buildAgentContext(page, currentStepGoal, stepNumber, totalSteps, scenarioSteps, memory, previousReflection, consecutiveFailures, noteFromPreviousIteration // NEW
+    async buildAgentContext(page, currentStepGoal, stepNumber, totalSteps, scenarioSteps, memory, consecutiveFailures, noteFromPreviousIteration, priorSteps, // NEW: For repair mode
+    nextSteps // NEW: For repair mode
     ) {
         // Get fresh DOM
         const currentPageInfo = await (0, page_info_utils_1.getEnhancedPageInfo)(page);
         const currentURL = page.url();
         // Get recent steps
         const recentSteps = memory.history.slice(-this.config.recentStepsCount);
+        // SoM integration: Update markers and capture screenshot with visual IDs
+        let somScreenshot = undefined;
+        let somElementMap = undefined;
+        if (this.config.useSoM && this.somHandler) {
+            try {
+                if (!this.somHandler) {
+                    this.somHandler = new page_som_handler_1.PageSoMHandler(page, this.logger);
+                }
+                else {
+                    this.somHandler.setPage(page);
+                }
+                // Wait briefly for page stability (handles first iteration + safety net for fast SPAs)
+                try {
+                    await page.waitForLoadState('domcontentloaded', { timeout: 5000 });
+                }
+                catch (error) {
+                    // Page already loaded or timeout - continue
+                }
+                // Update SoM markers
+                await this.somHandler.updateSom();
+                // Get screenshot WITH markers (viewport only - agent can scroll or use take_screenshot for full page)
+                somScreenshot = await this.somHandler.getScreenshot(true, false, 60);
+                // Get element map for disambiguation
+                somElementMap = this.somHandler.getSomElementMap();
+                this.logger?.(`[Orchestrator] SoM screenshot captured for agent decision-making`, 'log');
+            }
+            catch (error) {
+                this.logger?.(`[Orchestrator] Failed to capture SoM screenshot: ${error.message}`, 'error');
+            }
+        }
         // Build context
-        return {
+        const context = {
             overallGoal: scenarioSteps.join('\n'),
             currentStepGoal,
             stepNumber,
@@ -332,9 +391,17 @@ class OrchestratorAgent {
             recentSteps,
             experiences: memory.experiences,
             extractedData: memory.extractedData,
-            previousIterationGuidance: previousReflection,
-            noteFromPreviousIteration // NEW: Pass tactical note from previous iteration
+            noteFromPreviousIteration, // Pass tactical note from previous iteration
+            somScreenshot, // SoM screenshot with visual markers (current)
+            somElementMap, // SoM element details for disambiguation
+            priorSteps, // NEW: Repair context (undefined for script gen)
+            nextSteps // NEW: Repair context (undefined for script gen)
         };
+        // Save current screenshot as previous for next iteration (for tool access)
+        if (somScreenshot) {
+            this.previousSomScreenshot = somScreenshot;
+        }
+        return context;
     }
     /**
      * Call agent to make decision
@@ -346,10 +413,20 @@ class OrchestratorAgent {
         const useCoordinateMode = consecutiveFailures !== undefined && consecutiveFailures >= 3;
         // Build appropriate system prompt based on mode
         const toolDescriptions = this.toolRegistry.generateToolDescriptions();
-        const systemPrompt = useCoordinateMode
-            ? orchestrator_prompts_1.OrchestratorPrompts.buildCoordinateSystemPrompt()
-            : orchestrator_prompts_1.OrchestratorPrompts.buildSystemPrompt(toolDescriptions);
-        const userPrompt = orchestrator_prompts_1.OrchestratorPrompts.buildUserPrompt(context, consecutiveFailures);
+        let systemPrompt;
+        if (this.config.useSoM) {
+            // SoM mode: Use visual element identification
+            systemPrompt = orchestrator_prompts_1.OrchestratorPrompts.buildSomSystemPrompt(this.config.somRestrictCoordinates);
+        }
+        else if (useCoordinateMode) {
+            // Coordinate mode: Fallback when selectors fail
+            systemPrompt = orchestrator_prompts_1.OrchestratorPrompts.buildCoordinateSystemPrompt();
+        }
+        else {
+            // Standard mode: DOM-based selectors
+            systemPrompt = orchestrator_prompts_1.OrchestratorPrompts.buildSystemPrompt(toolDescriptions, this.config.enableCoordinateMode);
+        }
+        const userPrompt = orchestrator_prompts_1.OrchestratorPrompts.buildUserPrompt(context, consecutiveFailures, this.config.enableCoordinateMode);
         // Log prompt lengths for monitoring
         const systemLength = systemPrompt.length;
         const userLength = userPrompt.length;
@@ -363,6 +440,11 @@ class OrchestratorAgent {
                 systemPrompt,
                 userPrompt
             };
+            // Include current SoM screenshot as image
+            if (context.somScreenshot) {
+                llmRequest.imageUrl = context.somScreenshot;
+                this.logger?.(`[Orchestrator] Including SoM screenshot in LLM request`, 'log');
+            }
             const response = await this.llmFacade.llmProvider.callLLM(llmRequest);
             // Report token usage
             if (response.usage && this.progressReporter?.onTokensUsed) {
@@ -383,7 +465,7 @@ class OrchestratorAgent {
                 this.logger?.(`[Orchestrator] ⚠ No usage data in LLM response`, 'warn');
             }
             // Parse response
-            return this.parseAgentDecision(response.answer);
+            return this.decisionParser.parse(response.answer);
         }
         catch (error) {
             this.logger?.(`[Orchestrator] ✗ Agent call failed: ${error.message}`, 'error');
@@ -395,59 +477,20 @@ class OrchestratorAgent {
             };
         }
     }
-    /**
-     * Parse agent decision from LLM response
-     */
-    parseAgentDecision(response) {
-        try {
-            // Extract JSON from response
-            const jsonMatch = response.match(/\{[\s\S]*\}/);
-            if (!jsonMatch) {
-                this.logger?.(`[Orchestrator] ✗ No JSON found in LLM response`, 'error');
-                this.logger?.(`[Orchestrator] 📄 FULL LLM RESPONSE:\n${response}`, 'error');
-                throw new Error('No JSON found in response');
-            }
-            const parsed = JSON.parse(jsonMatch[0]);
-            // Validate required fields
-            // Accept either "reasoning" or "statusReasoning" (LLMs sometimes only provide one)
-            if (!parsed.status || (!parsed.reasoning && !parsed.statusReasoning)) {
-                this.logger?.(`[Orchestrator] ✗ Missing required fields in parsed JSON`, 'error');
-                this.logger?.(`[Orchestrator] 📄 FULL LLM RESPONSE:\n${response}`, 'error');
-                this.logger?.(`[Orchestrator] 📄 PARSED JSON:\n${JSON.stringify(parsed, null, 2)}`, 'error');
-                this.logger?.(`[Orchestrator] ❌ Has status: ${!!parsed.status}, Has reasoning: ${!!parsed.reasoning}, Has statusReasoning: ${!!parsed.statusReasoning}`, 'error');
-                throw new Error('Missing required fields: status and (reasoning or statusReasoning)');
-            }
-            // Normalize: if reasoning is missing but statusReasoning exists, use statusReasoning as reasoning
-            if (!parsed.reasoning && parsed.statusReasoning) {
-                parsed.reasoning = parsed.statusReasoning;
-            }
-            return parsed;
-        }
-        catch (error) {
-            this.logger?.(`[Orchestrator] ✗ Failed to parse agent decision: ${error.message}`, 'error');
-            // Only log full response if not already logged above
-            if (!error.message.includes('Missing required fields') && !error.message.includes('No JSON found')) {
-                this.logger?.(`[Orchestrator] 📄 FULL LLM RESPONSE:\n${response}`, 'error');
-            }
-            // Return fallback
-            return {
-                status: 'stuck',
-                statusReasoning: 'Failed to parse agent response',
-                reasoning: `Parse error: ${error.message}`
-            };
-        }
-    }
     /**
      * Execute tools
      */
-    async executeTools(toolCalls, page, memory, stepNumber) {
+    async executeTools(toolCalls, page, memory, stepNumber, refMap) {
         this.logger?.(`[Orchestrator] 🔧 Executing ${toolCalls.length} tool(s)`);
         const results = {};
         const toolContext = {
             page,
             memory,
             stepNumber,
-            logger: this.logger
+            logger: this.logger,
+            refMap, // Pass refMap for interact_with_ref tool
+            previousSomScreenshot: this.previousSomScreenshot, // For view_previous_screenshot tool
+            somHandler: this.somHandler // For refresh_som_markers tool
         };
         for (const toolCall of toolCalls.slice(0, this.config.maxToolCallsPerIteration)) {
             this.logger?.(`[Orchestrator]   ▶ ${toolCall.name}(${JSON.stringify(toolCall.params).substring(0, 50)}...)`);
@@ -463,45 +506,214 @@ class OrchestratorAgent {
         return results;
     }
     /**
-     * Execute commands sequentially with SHARED context (variables persist across commands)
+     * Parse SomCommand from command object
      */
-    async executeCommandsSequentially(commands, page, memory, stepNumber, iteration, jobId) {
-        this.logger?.(`[Orchestrator] 📝 Executing ${commands.length} command(s) in shared context`);
+    parseSomCommand(cmd) {
+        if (typeof cmd === 'object' && cmd.action) {
+            // Valid if: has elementRef, OR has coord, OR is navigation action
+            const isNavigationAction = ['navigate', 'goBack', 'goForward', 'reload'].includes(cmd.action);
+            const hasValidTarget = cmd.elementRef || cmd.coord || isNavigationAction;
+            if (hasValidTarget) {
+                return {
+                    elementRef: cmd.elementRef,
+                    coord: cmd.coord,
+                    action: cmd.action,
+                    value: cmd.value,
+                    fromCoord: cmd.fromCoord,
+                    toCoord: cmd.toCoord,
+                    force: cmd.force,
+                    scrollAmount: cmd.scrollAmount,
+                    scrollDirection: cmd.scrollDirection,
+                    button: cmd.button,
+                    clickCount: cmd.clickCount,
+                    modifiers: cmd.modifiers,
+                    delay: cmd.delay,
+                    timeout: cmd.timeout
+                };
+            }
+        }
+        return null;
+    }
+    /**
+     * Execute commands (mix of ref and playwright commands)
+     */
+    async executeCommands(commands, page, memory, stepNumber, iteration, jobId) {
+        this.logger?.(`[Orchestrator] 📝 Executing ${commands.length} command(s)`);
         const executed = [];
-        const limitedCommands = commands.slice(0, this.config.maxCommandsPerIteration);
-        // Build execution with shared context (all commands share scope - variables persist)
-        const commandsWithTracking = limitedCommands.map((cmd, i) => {
-            return `
-// Command ${i + 1}/${limitedCommands.length}
+        if (commands.length === 0) {
+            return { executed: [], allSucceeded: true };
+        }
+        // SoM mode: Execute commands through PageSoMHandler
+        if (this.config.useSoM && this.somHandler) {
+            this.logger?.(`[Orchestrator] Using SoM mode for command execution`, 'log');
+            for (let i = 0; i < commands.length; i++) {
+                const cmd = commands[i];
+                // Check if verification or action command
+                if ((0, som_types_1.isSomVerification)(cmd)) {
+                    // Handle verification command
+                    try {
+                        const result = await this.somHandler.executeVerification(cmd);
+                        // Always add command to executed array (even if verification failed)
+                        // Scripts should contain the expect even if it fails during generation
+                        if (result.playwrightCommand) {
+                            executed.push(result.playwrightCommand);
+                        }
+                        if (result.success) {
+                            this.logger?.(`[Orchestrator]   ✓ [${i + 1}/${commands.length}] Verification passed`, 'log');
+                            memory.history.push({
+                                stepNumber,
+                                iteration,
+                                action: `Verification ${i + 1}/${commands.length}: ${cmd.verificationType}`,
+                                code: result.playwrightCommand,
+                                result: 'success',
+                                observation: `Verified: ${cmd.description || cmd.expected}`,
+                                url: page.url(),
+                                timestamp: Date.now()
+                            });
+                        }
+                        else {
+                            this.logger?.(`[Orchestrator]   ✗ [${i + 1}/${commands.length}] Verification failed (non-fatal): ${result.error}`, 'warn');
+                            memory.history.push({
+                                stepNumber,
+                                iteration,
+                                action: `Verification ${i + 1}/${commands.length} - FAILED`,
+                                code: result.playwrightCommand || JSON.stringify(cmd),
+                                result: 'failure',
+                                observation: `Failed: ${result.error}`,
+                                error: result.error,
+                                url: page.url(),
+                                timestamp: Date.now()
+                            });
+                            // Continue anyway - verification failures are non-blocking for script generation
+                        }
+                        // Small delay between commands
+                        if (i < commands.length - 1) {
+                            await page.waitForTimeout(300);
+                        }
+                    }
+                    catch (error) {
+                        this.logger?.(`[Orchestrator]   ✗ [${i + 1}/${commands.length}] Verification exception: ${error.message}`, 'error');
+                    }
+                }
+                else if ((0, som_types_1.isSomCommand)(cmd)) {
+                    // Handle action command (existing logic)
+                    const somCommand = cmd;
+                    try {
+                        const result = await this.somHandler.runCommand(somCommand, this.config.somUseSomIdBasedCommands || false);
+                        if (result.status === som_types_1.CommandRunStatus.SUCCESS && result.successAttempt) {
+                            this.logger?.(`[Orchestrator]   ✓ [${i + 1}/${commands.length}] SoM action succeeded`, 'log');
+                            executed.push(result.successAttempt.command);
+                            memory.history.push({
+                                stepNumber,
+                                iteration,
+                                action: `SoM Action ${i + 1}/${commands.length}: ${somCommand.action}`,
+                                code: result.successAttempt.command,
+                                result: 'success',
+                                observation: 'Executed successfully',
+                                url: page.url(),
+                                timestamp: Date.now()
+                            });
+                            // Small delay for form validation/animations
+                            if (i < commands.length - 1) {
+                                await page.waitForTimeout(300);
+                            }
+                        }
+                        else {
+                            this.logger?.(`[Orchestrator]   ✗ [${i + 1}/${commands.length}] SoM action failed: ${result.error}`, 'error');
+                            memory.history.push({
+                                stepNumber,
+                                iteration,
+                                action: `SoM Action ${i + 1}/${commands.length}: ${somCommand.action} - FAILED`,
+                                code: JSON.stringify(somCommand),
+                                result: 'failure',
+                                observation: `Failed: ${result.error}`,
+                                error: result.error,
+                                url: page.url(),
+                                timestamp: Date.now()
+                            });
+                            // Refresh SoM after batch (DOM may have changed)
+                            if (this.somHandler && page) {
+                                this.somHandler.setPage(page);
+                                await this.somHandler.updateSom();
+                            }
+                            return { executed, allSucceeded: false };
+                        }
+                    }
+                    catch (error) {
+                        this.logger?.(`[Orchestrator]   ✗ [${i + 1}/${commands.length}] SoM action exception: ${error.message}`, 'error');
+                        memory.history.push({
+                            stepNumber,
+                            iteration,
+                            action: `SoM Action ${i + 1}/${commands.length} - EXCEPTION`,
+                            code: JSON.stringify(somCommand),
+                            result: 'failure',
+                            observation: `Exception: ${error.message}`,
+                            error: error.message,
+                            url: page.url(),
+                            timestamp: Date.now()
+                        });
+                        // Refresh SoM after batch (DOM may have changed)
+                        if (this.somHandler && page) {
+                            this.somHandler.setPage(page);
+                            await this.somHandler.updateSom();
+                        }
+                        return { executed, allSucceeded: false };
+                    }
+                }
+                else {
+                    this.logger?.(`[Orchestrator]   ⚠ [${i + 1}/${commands.length}] Not a valid SoM command/verification, skipping`, 'warn');
+                }
+            }
+            // Always wait for page to stabilize after command batch
+            // This handles both explicit navigation AND clicks that trigger navigation/SPA routes
+            try {
+                this.logger?.(`[Orchestrator] Waiting for page to stabilize...`, 'log');
+                // Use networkidle with short timeout to catch navigation without blocking on SPAs with continuous requests
+                await page.waitForLoadState('networkidle', { timeout: 3000 });
+                this.logger?.(`[Orchestrator] Page stabilized (networkidle)`, 'log');
+            }
+            catch (error) {
+                // If networkidle times out, fall back to domcontentloaded
+                try {
+                    await page.waitForLoadState('domcontentloaded', { timeout: 2000 });
+                    this.logger?.(`[Orchestrator] Page loaded (domcontentloaded)`, 'log');
+                }
+                catch (error2) {
+                    this.logger?.(`[Orchestrator] Page load wait timeout (continuing anyway)`, 'warn');
+                }
+            }
+            // Refresh SoM after batch (DOM may have changed and page is now stable)
+            if (this.somHandler && page) {
+                this.somHandler.setPage(page);
+                await this.somHandler.updateSom();
+            }
+            return { executed, allSucceeded: true };
+        }
+        // Standard mode: Execute all commands in sequence with small delay between them
+        // Delay helps with form validation, button enabling, and animations
+        const wrappedCode = commands.map((cmd, i) => `
+// Command ${i + 1}/${commands.length}
 try {
   ${cmd}
   __results.push({ index: ${i}, success: true });
+  ${i < commands.length - 1 ? 'await page.waitForTimeout(300);' : ''} // Small delay for form validation/animations
 } catch (error) {
   __results.push({ index: ${i}, success: false, error: error.message });
-  throw error; // Stop on first failure
-}`;
-        }).join('\n');
-        const wrappedCode = `
-const __results = [];
-${commandsWithTracking}
-return __results;
-`;
+  throw error;
+}`).join('\n');
+        const fullCode = `const __results = []; ${wrappedCode} return __results;`;
         try {
-            // Execute in shared context - variables declared here persist for entire scenario
-            // Pass both page and expect to make Playwright assertions available
-            const func = new Function('page', 'expect', 'return (async () => { ' + wrappedCode + ' })()');
-            const results = await func(page, test_1.expect);
-            // Record results for each command
-            for (let i = 0; i < limitedCommands.length; i++) {
-                const cmd = limitedCommands[i];
+            const func = new Function('page', 'expect', 'return (async () => { ' + fullCode + ' })()');
+            const results = await func(page, global.expect);
+            for (let i = 0; i < commands.length; i++) {
+                const cmd = commands[i];
                 const result = results[i];
                 if (result && result.success) {
-                    this.logger?.(`[Orchestrator]   ✓ [${i + 1}/${limitedCommands.length}] Success`);
-                    // Record in history
+                    this.logger?.(`[Orchestrator]   ✓ [${i + 1}/${commands.length}] Success`);
                     memory.history.push({
                         stepNumber,
                         iteration,
-                        action: `Command ${i + 1}/${limitedCommands.length}`,
+                        action: `Command ${i + 1}/${commands.length}`,
                         code: cmd,
                         result: 'success',
                         observation: 'Executed successfully',
@@ -511,122 +723,27 @@ return __results;
                     executed.push(cmd);
                 }
             }
-            // Cap history
             if (memory.history.length > this.config.maxHistorySize) {
                 memory.history = memory.history.slice(-this.config.maxHistorySize);
             }
             return { executed, allSucceeded: true };
         }
         catch (error) {
-            // One of the commands failed - find which one
             const errorMessage = error.message || String(error);
-            // Capture page state for debug logging
-            let pageStateDebug = '';
-            if (this.debugMode) {
-                try {
-                    const pageInfo = await (0, page_info_utils_1.getEnhancedPageInfo)(page);
-                    pageStateDebug = `
-=== DEBUG: PAGE STATE AT FAILURE ===
-URL: ${page.url()}
-Title: ${pageInfo.title}
-
-INTERACTIVE ELEMENTS:
-${pageInfo.formattedElements}
-
-ARIA SNAPSHOT:
-${JSON.stringify(pageInfo.ariaSnapshot, null, 2)}
-
-====================================`;
-                }
-                catch (debugError) {
-                    pageStateDebug = `Failed to capture page state: ${debugError.message}`;
-                }
-            }
-            // Record all that succeeded, then the failure
-            for (let i = 0; i < limitedCommands.length; i++) {
-                const cmd = limitedCommands[i];
-                // This is a failed command (error happened here or earlier)
-                if (executed.length <= i) {
-                    this.logger?.(`[Orchestrator]   ✗ [${i + 1}/${limitedCommands.length}] Failed: ${errorMessage}`, 'error');
-                    // Log detailed debug info
-                    if (this.debugMode && pageStateDebug) {
-                        this.logger?.(pageStateDebug, 'debug');
-                    }
-                    memory.history.push({
-                        stepNumber,
-                        iteration,
-                        action: `Command ${i + 1}/${limitedCommands.length} - FAILED`,
-                        code: cmd,
-                        result: 'failure',
-                        observation: `Failed with error: ${errorMessage}. This selector likely doesn't exist or is incorrect.`,
-                        error: errorMessage,
-                        url: page.url(),
-                        timestamp: Date.now()
-                    });
-                    if (i < limitedCommands.length - 1) {
-                        this.logger?.(`[Orchestrator]   ⚠ Skipping remaining ${limitedCommands.length - i - 1} command(s)`, 'warn');
-                    }
-                    break;
-                }
-            }
-            return { executed, allSucceeded: false };
-        }
-    }
-    /**
-     * Execute a single command
-     */
-    async executeCommand(cmd, page) {
-        // Wrap in async function and execute
-        const wrapped = `(async () => { ${cmd} })()`;
-        try {
-            await eval(wrapped);
-        }
-        catch (error) {
-            // If eval fails, try direct execution with page context
-            // Pass both page and expect to make Playwright assertions available
-            const func = new Function('page', 'expect', `return (async () => { ${cmd} })()`);
-            await func(page, test_1.expect);
-        }
-    }
-    /**
-     * Log agent decision
-     */
-    logAgentDecision(decision, iteration) {
-        this.logger?.(`[Orchestrator] 💭 REASONING: ${decision.reasoning}`);
-        if (decision.selfReflection) {
-            this.logger?.(`[Orchestrator] 🧠 SELF-REFLECTION:`);
-            this.logger?.(`[Orchestrator]    Next: ${decision.selfReflection.guidanceForNext}`);
-            if (decision.selfReflection.detectingLoop) {
-                this.logger?.(`[Orchestrator]    🔄 LOOP DETECTED: ${decision.selfReflection.loopReasoning}`, 'warn');
-            }
-        }
-        if (decision.toolCalls && decision.toolCalls.length > 0) {
-            this.logger?.(`[Orchestrator] 🔧 TOOLS: ${decision.toolCalls.map(t => t.name).join(', ')}`);
-            if (decision.toolReasoning) {
-                this.logger?.(`[Orchestrator] 📋 Why: ${decision.toolReasoning}`);
-            }
-        }
-        if (decision.blockerDetected) {
-            this.logger?.(`[Orchestrator] 🚧 BLOCKER: ${decision.blockerDetected.description}`, 'warn');
-            this.logger?.(`[Orchestrator] 🧹 Clearing with ${decision.blockerDetected.clearingCommands.length} command(s)`);
-        }
-        if (decision.stepReEvaluation?.detected) {
-            this.logger?.(`[Orchestrator] 🔍 STEP RE-EVALUATION: ${decision.stepReEvaluation.issue}`, 'warn');
-            this.logger?.(`[Orchestrator] 📝 Explanation: ${decision.stepReEvaluation.explanation}`);
-        }
-        if (decision.commands && decision.commands.length > 0) {
-            this.logger?.(`[Orchestrator] 📝 COMMANDS (${decision.commands.length}):`);
-            decision.commands.slice(0, 3).forEach((cmd, i) => {
-                this.logger?.(`[Orchestrator]    ${i + 1}. ${cmd.substring(0, 80)}...`);
+            this.logger?.(`[Orchestrator] ❌ Command execution failed: ${errorMessage}`, 'error');
+            memory.history.push({
+                stepNumber,
+                iteration,
+                action: `Command - FAILED`,
+                code: commands[executed.length] || '',
+                result: 'failure',
+                observation: `Failed: ${errorMessage}`,
+                error: errorMessage,
+                url: page.url(),
+                timestamp: Date.now()
             });
-            if (decision.commands.length > 3) {
-                this.logger?.(`[Orchestrator]    ... and ${decision.commands.length - 3} more`);
-            }
-            if (decision.commandReasoning) {
-                this.logger?.(`[Orchestrator] 💡 Why: ${decision.commandReasoning}`);
-            }
+            return { executed, allSucceeded: false };
         }
-        // Experiences will be logged when added to memory, no need to log here
     }
     /**
      * Report step progress
@@ -651,6 +768,219 @@ ${JSON.stringify(pageInfo.ariaSnapshot, null, 2)}
             agentStatus: decision.status
         });
     }
+    /**
+     * Execute exploration mode - agent autonomously explores to achieve journey goal
+     * Fires onStepProgress callbacks for each autonomous action (transparent to caller)
+     */
+    async executeExploration(page, explorationConfig, jobId) {
+        this.logger?.(`\n[Orchestrator] ========== EXPLORATION MODE ==========`);
+        this.logger?.(`[Orchestrator] 🎯 Journey Goal: ${explorationConfig.explorationPrompt}`);
+        if (explorationConfig.testDataPrompt) {
+            this.logger?.(`[Orchestrator] 📋 Test Data: ${explorationConfig.testDataPrompt}`);
+        }
+        const memory = {
+            history: [],
+            experiences: [],
+            extractedData: {}
+        };
+        const maxSteps = explorationConfig.maxExplorationSteps || 50;
+        let stepNumber = 0;
+        const commandsExecuted = [];
+        while (stepNumber < maxSteps) {
+            stepNumber++;
+            this.logger?.(`\n[Orchestrator] === Exploration Step ${stepNumber}/${maxSteps} ===`);
+            // Build exploratory context
+            const context = await this.buildExploratoryContext(page, explorationConfig.explorationPrompt, explorationConfig.testDataPrompt, memory, stepNumber, maxSteps);
+            // Call agent with exploratory prompt
+            const decision = await this.callExploratoryAgent(context, jobId, stepNumber);
+            this.decisionParser.log(decision, stepNumber);
+            // Report step start (fires JourneyRunner's beforeStepStart callback)
+            if (this.progressReporter?.onStepProgress) {
+                const stepInfo = {
+                    jobId,
+                    stepNumber,
+                    stepId: `exploration-${stepNumber}-${Date.now()}`,
+                    description: decision.reasoning,
+                    code: '', // Will be filled after commands execute
+                    status: progress_reporter_1.StepExecutionStatus.IN_PROGRESS,
+                    wasRepaired: false
+                };
+                await this.progressReporter.onStepProgress(stepInfo);
+            }
+            // Execute tools if requested
+            if (decision.toolCalls && decision.toolCalls.length > 0) {
+                const toolResults = await this.executeTools(decision.toolCalls, page, memory, stepNumber);
+                // If needs tool results, call agent again
+                if (decision.needsToolResults) {
+                    const updatedContext = { ...context, toolResults };
+                    const continuedDecision = await this.callExploratoryAgent(updatedContext, jobId, stepNumber);
+                    decision.commands = continuedDecision.commands || decision.commands;
+                    decision.commandReasoning = continuedDecision.commandReasoning || decision.commandReasoning;
+                    decision.status = continuedDecision.status;
+                }
+            }
+            // Handle blocker clearing
+            if (decision.blockerDetected && decision.blockerDetected.clearingCommands) {
+                this.logger?.(`[Orchestrator] 🚧 Clearing blocker: ${decision.blockerDetected.description}`);
+                const blockerResult = await this.executeCommands(decision.blockerDetected.clearingCommands, page, memory, stepNumber, 1, jobId);
+                commandsExecuted.push(...blockerResult.executed);
+            }
+            // Execute exploration commands
+            let commandsSucceeded = true;
+            if (decision.commands && decision.commands.length > 0) {
+                const executeResult = await this.executeCommands(decision.commands, page, memory, stepNumber, 1, jobId);
+                commandsExecuted.push(...executeResult.executed);
+                commandsSucceeded = executeResult.allSucceeded;
+            }
+            // Report step completion (fires JourneyRunner's onStepComplete callback)
+            if (this.progressReporter?.onStepProgress) {
+                const stepInfo = {
+                    jobId,
+                    stepNumber,
+                    stepId: `exploration-${stepNumber}-${Date.now()}`,
+                    description: decision.reasoning,
+                    code: decision.commands?.join('\n') || '',
+                    status: commandsSucceeded ? progress_reporter_1.StepExecutionStatus.SUCCESS : progress_reporter_1.StepExecutionStatus.FAILURE,
+                    error: commandsSucceeded ? undefined : 'Command execution failed',
+                    wasRepaired: false
+                };
+                await this.progressReporter.onStepProgress(stepInfo);
+            }
+            // Add experiences (both app patterns AND exploration progress)
+            if (decision.experiences) {
+                memory.experiences.push(...decision.experiences);
+                if (memory.experiences.length > this.config.maxExperiences) {
+                    memory.experiences = memory.experiences.slice(-this.config.maxExperiences);
+                }
+            }
+            // Store note for next iteration
+            if (decision.noteToFutureSelf) {
+                memory.latestNote = {
+                    fromIteration: stepNumber,
+                    content: decision.noteToFutureSelf
+                };
+            }
+            // Check termination
+            if (decision.status === 'complete') {
+                this.logger?.(`[Orchestrator] ✅ Journey exploration complete: ${decision.statusReasoning}`);
+                return {
+                    success: true,
+                    commands: commandsExecuted,
+                    iterations: stepNumber,
+                    terminationReason: 'complete',
+                    memory
+                };
+            }
+            else if (decision.status === 'stuck') {
+                this.logger?.(`[Orchestrator] ❌ Exploration stuck: ${decision.statusReasoning}`);
+                return {
+                    success: false,
+                    commands: commandsExecuted,
+                    iterations: stepNumber,
+                    terminationReason: 'agent_stuck',
+                    memory,
+                    error: decision.statusReasoning
+                };
+            }
+        }
+        // Hit max steps - not necessarily a failure
+        this.logger?.(`[Orchestrator] ⚠ Maximum exploration steps reached (budget limit)`);
+        return {
+            success: true, // Not a failure - just budget limit
+            commands: commandsExecuted,
+            iterations: stepNumber,
+            terminationReason: 'system_limit',
+            memory
+        };
+    }
+    async buildExploratoryContext(page, explorationPrompt, testDataPrompt, memory, stepNumber, maxSteps) {
+        // Wait for page to be ready and elements to appear (especially important after navigation)
+        const currentPageInfo = await page_info_retry_1.PageInfoRetry.getWithRetry(page);
+        const currentURL = page.url();
+        const recentSteps = memory.history.slice(-this.config.recentStepsCount);
+        // SoM integration for exploratory mode
+        let somScreenshot = undefined;
+        let somElementMap = undefined;
+        if (this.config.useSoM && this.somHandler) {
+            try {
+                this.somHandler.setPage(page);
+                // Wait briefly for page stability (handles first iteration + safety net for fast SPAs)
+                try {
+                    await page.waitForLoadState('domcontentloaded', { timeout: 2000 });
+                }
+                catch (error) {
+                    // Page already loaded or timeout - continue
+                }
+                // Update SoM markers
+                await this.somHandler.updateSom();
+                somScreenshot = await this.somHandler.getScreenshot(true, false, 60); // Viewport only - agent can scroll or request full page
+                // Get element map for disambiguation
+                somElementMap = this.somHandler.getSomElementMap();
+                this.logger?.(`[Orchestrator] SoM screenshot captured for exploratory agent`, 'log');
+            }
+            catch (error) {
+                this.logger?.(`[Orchestrator] Failed to capture SoM screenshot: ${error.message}`, 'error');
+            }
+        }
+        const context = {
+            overallGoal: explorationPrompt,
+            currentStepGoal: explorationPrompt, // Same as overall for single journey
+            stepNumber,
+            totalSteps: maxSteps,
+            completedSteps: [],
+            remainingSteps: [],
+            currentPageInfo,
+            currentURL,
+            recentSteps,
+            experiences: memory.experiences,
+            extractedData: memory.extractedData,
+            noteFromPreviousIteration: memory.latestNote,
+            testDataPrompt, // CRITICAL: Store testDataPrompt in context
+            somScreenshot, // SoM screenshot for exploratory mode (current)
+            somElementMap // SoM element details for disambiguation
+        };
+        // Save current screenshot as previous for next iteration (for tool access)
+        if (somScreenshot) {
+            this.previousSomScreenshot = somScreenshot;
+        }
+        return context;
+    }
+    async callExploratoryAgent(context, jobId, stepNumber) {
+        const toolDescriptions = this.toolRegistry.generateToolDescriptions();
+        // Use SoM system prompt if in SoM mode, otherwise use standard exploratory prompt
+        const systemPrompt = this.config.useSoM
+            ? orchestrator_prompts_1.OrchestratorPrompts.buildSomSystemPrompt(this.config.somRestrictCoordinates)
+            : orchestrator_prompts_1.OrchestratorPrompts.buildExploratorySystemPrompt(toolDescriptions);
+        const userPrompt = orchestrator_prompts_1.OrchestratorPrompts.buildExploratoryUserPrompt(context, context.overallGoal, context.testDataPrompt, // Pass testDataPrompt from context
+        stepNumber, context.totalSteps);
+        const llmRequest = {
+            model: model_constants_1.DEFAULT_MODEL,
+            systemPrompt,
+            userPrompt
+        };
+        // Include current SoM screenshot as image
+        if (context.somScreenshot) {
+            llmRequest.imageUrl = context.somScreenshot;
+            this.logger?.(`[Orchestrator] Including SoM screenshot in exploratory LLM request`, 'log');
+        }
+        const response = await this.llmFacade.llmProvider.callLLM(llmRequest);
+        // Report token usage
+        if (response.usage && this.progressReporter?.onTokensUsed) {
+            await this.progressReporter.onTokensUsed({
+                jobId,
+                stepNumber,
+                iteration: 1,
+                inputTokens: response.usage.inputTokens,
+                outputTokens: response.usage.outputTokens,
+                includesImage: false,
+                model: model_constants_1.DEFAULT_MODEL,
+                timestamp: Date.now()
+            });
+        }
+        // Parse response (same JSON format as regular mode)
+        const decision = this.decisionParser.parse(response.answer);
+        return decision;
+    }
 }
 exports.OrchestratorAgent = OrchestratorAgent;
 //# sourceMappingURL=orchestrator-agent.js.map