testchimp-runner-core 0.0.35 → 0.0.37

package/src/orchestrator/orchestrator-agent.ts

@@ -1,1282 +0,0 @@
-/**
- * Orchestrator Agent
- * Single agent with tool-use capabilities that maintains journey memory and executes scenarios
- */
-
-import { expect } from '@playwright/test';
-import { LLMFacade } from '../llm-facade';
-import { ProgressReporter, TokenUsage, StepExecutionStatus } from '../progress-reporter';
-import { getEnhancedPageInfo, PageInfo } from '../utils/page-info-utils';
-import { CoordinateConverter } from '../utils/coordinate-converter';
-import { ToolRegistry, ToolExecutionContext } from './tool-registry';
-import { DEFAULT_MODEL } from '../model-constants';
-import {
-  AgentConfig,
-  AgentContext,
-  AgentDecision,
-  JourneyMemory,
-  MemoryStep,
-  OrchestratorStepResult,
-  SelfReflection,
-  NoteToFutureSelf,
-  CoordinateAction,
-  ExplorationMode,
-  DEFAULT_AGENT_CONFIG
-} from './types';
-import { OrchestratorPrompts } from './orchestrator-prompts';
-import { PageInfoRetry } from '../utils/page-info-retry';
-import { DecisionParser } from './decision-parser';
-import { PageSoMHandler } from './page-som-handler';
-import { SomCommand, CommandRunStatus, InteractionAction, isSomVerification, isSomCommand, SomVerification } from './som-types';
-
-/**
- * Orchestrator Agent - manages step execution with tool use and memory
- */
-export class OrchestratorAgent {
-  private llmFacade: LLMFacade;
-  private toolRegistry: ToolRegistry;
-  private progressReporter?: ProgressReporter;
-  private config: Required<AgentConfig>;
-  private logger?: (message: string, level?: 'log' | 'error' | 'warn' | 'debug') => void;
-  private debugMode: boolean = false;
-  private decisionParser: DecisionParser;
-  private somHandler?: PageSoMHandler;
-  private previousSomScreenshot?: string;  // Track previous iteration's screenshot
-  
-  constructor(
-    llmFacade: LLMFacade,
-    toolRegistry: ToolRegistry,
-    config?: Partial<AgentConfig>,
-    progressReporter?: ProgressReporter,
-    logger?: (message: string, level?: 'log' | 'error' | 'warn' | 'debug') => void,
-    debugMode?: boolean
-  ) {
-    this.llmFacade = llmFacade;
-    this.toolRegistry = toolRegistry;
-    this.config = { ...DEFAULT_AGENT_CONFIG, ...config };
-    this.progressReporter = progressReporter;
-    this.logger = logger;
-    this.debugMode = debugMode || false;
-    this.decisionParser = new DecisionParser(logger);
-    
-    // Initialize SoM handler if enabled
-    if (this.config.useSoM) {
-      this.somHandler = new PageSoMHandler(null as any, this.logger);
-    }
-  }
-  
-  setDebugMode(enabled: boolean): void {
-    this.debugMode = enabled;
-  }
-  
-  /**
-   * Execute a single step of the scenario
-   */
-  async executeStep(
-    page: any,
-    stepDescription: string,
-    stepNumber: number,
-    totalSteps: number,
-    scenarioSteps: string[],
-    memory: JourneyMemory,
-    jobId: string,
-    priorSteps?: string[],    // NEW: For repair mode (undefined for script gen)
-    nextSteps?: string[]      // NEW: For repair mode (undefined for script gen)
-  ): Promise<OrchestratorStepResult> {
-    this.logger?.(`\n[Orchestrator] ========== STEP ${stepNumber}/${totalSteps} ==========`);
-    this.logger?.(`[Orchestrator] 🎯 Goal: ${stepDescription}`);
-    
-    let iteration = 0;
-    let noteToSelf: NoteToFutureSelf | undefined = memory.latestNote;  // Start with note from previous step
-    const commandsExecuted: string[] = [];
-    let consecutiveFailures = 0;  // Track consecutive iterations with failed commands
-    let coordinateAttempts = 0;  // Track coordinate mode attempts (max 2)
-    
-    while (iteration < this.config.maxIterationsPerStep) {
-      iteration++;
-      
-      this.logger?.(`\n[Orchestrator] === Iteration ${iteration}/${this.config.maxIterationsPerStep} ===`);
-      
-      // Build context for agent
-      const context = await this.buildAgentContext(
-        page,
-        stepDescription,
-        stepNumber,
-        totalSteps,
-        scenarioSteps,
-        memory,
-        consecutiveFailures,
-        noteToSelf,  // Pass note from previous iteration
-        priorSteps,  // NEW: Pass repair context
-        nextSteps    // NEW: Pass repair context
-      );
-      
-      // Call agent to make decision
-      const decision = await this.callAgent(
-        context,
-        jobId,
-        stepNumber,
-        iteration,
-        consecutiveFailures
-      );
-      
-      // Log agent's reasoning
-      this.decisionParser.log(decision, iteration);
-      
-      // Report progress
-      await this.reportStepProgress(jobId, stepNumber, stepDescription, decision, iteration);
-      
-      // Execute tools if requested (tools are READ-ONLY, they don't change state)
-      let toolResults: Record<string, any> = {};
-      
-      // ANTI-LOOP: Detect and BLOCK screenshot loops (PER STEP)
-      const screenshotsThisStep = memory.history.filter(s => 
-        s.stepNumber === stepNumber && 
-        (s.code.includes('take_screenshot') || s.action.includes('Screenshot'))
-      );
-      const recentScreenshots = memory.history.slice(-3).filter(s => 
-        s.code.includes('take_screenshot') || s.action.includes('Screenshot')
-      );
-      
-      if (screenshotsThisStep.length >= 3) {
-        this.logger?.(`[Orchestrator] 🚨 SCREENSHOT LOOP - ${screenshotsThisStep.length} screenshots THIS STEP! BLOCKING further screenshots`, 'error');
-      } else if (recentScreenshots.length >= 2 && iteration >= 3) {
-        this.logger?.(`[Orchestrator] ⚠️  WARNING: ${recentScreenshots.length} screenshots in last 3 iterations - agent may be looping`, 'warn');
-      }
-      
-      if (decision.toolCalls && decision.toolCalls.length > 0) {
-        // ENFORCE: Block screenshot tool calls if too many taken IN THIS STEP
-        if (screenshotsThisStep.length >= 3) {
-          decision.toolCalls = decision.toolCalls.filter(tc => tc.name !== 'take_screenshot');
-          if (decision.toolCalls.length === 0) {
-            this.logger?.(`[Orchestrator] 🚫 REJECTED screenshot tool call - loop detected. Agent must ACT.`, 'warn');
-            toolResults = [{
-              toolName: 'take_screenshot',
-              success: false,
-              error: 'SYSTEM BLOCKED: Too many screenshots taken. You must use existing DOM snapshots and execute commands now. Analysis paralysis detected.',
-              data: null
-            }];
-          }
-        }
-        
-        if (decision.toolCalls.length > 0) {
-          toolResults = await this.executeTools(decision.toolCalls, page, memory, stepNumber, context.currentPageInfo.refMap);
-        }
-        
-        // If agent wants to wait for tool results before proceeding, call agent again
-        if (decision.needsToolResults) {
-          const updatedContext = { ...context, toolResults };
-          const continuedDecision = await this.callAgent(updatedContext, jobId, stepNumber, iteration, consecutiveFailures);
-          
-          // Merge continued decision
-          decision.commands = continuedDecision.commands || decision.commands;
-          decision.commandReasoning = continuedDecision.commandReasoning || decision.commandReasoning;
-          decision.status = continuedDecision.status;
-          decision.statusReasoning = continuedDecision.statusReasoning;
-          decision.reasoning = continuedDecision.reasoning;
-        }
-      }
-      
-      // Execute commands sequentially
-      let iterationHadFailure = false;
-      
-      // Handle blocker if detected (clear blocker FIRST, then proceed with main commands)
-      if (decision.blockerDetected && decision.blockerDetected.clearingCommands && decision.blockerDetected.clearingCommands.length > 0) {
-        this.logger?.(`[Orchestrator] 🚧 BLOCKER DETECTED: ${decision.blockerDetected.description}`);
-        this.logger?.(`[Orchestrator] 🧹 Clearing blocker with ${decision.blockerDetected.clearingCommands.length} command(s)...`);
-        
-        const blockerResult = await this.executeCommands(
-          decision.blockerDetected.clearingCommands,
-          page,
-          memory,
-          stepNumber,
-          iteration,
-          jobId
-        );
-        
-        // Add blocker commands with comment to output
-        if (blockerResult.executed.length > 0) {
-          commandsExecuted.push(`// Blocker: ${decision.blockerDetected.description}`);
-          commandsExecuted.push(...blockerResult.executed);
-        }
-        
-        // If blocker clearing failed, track it
-        if (!blockerResult.allSucceeded) {
-          this.logger?.(`[Orchestrator] ❌ Failed to clear blocker - continuing anyway`);
-          consecutiveFailures++;
-          iterationHadFailure = true;
-        }
-      }
-      
-      // Execute main commands (only if no blocker failure)
-      if (!iterationHadFailure && decision.commands && decision.commands.length > 0) {
-        const executeResult = await this.executeCommands(
-          decision.commands,
-          page,
-          memory,
-          stepNumber,
-          iteration,
-          jobId
-        );
-        
-        commandsExecuted.push(...executeResult.executed);
-        
-        // Track failures
-        if (!executeResult.allSucceeded) {
-          this.logger?.(`[Orchestrator] ⚠ Command execution stopped at failure`);
-          consecutiveFailures++;
-          iterationHadFailure = true;
-        } else {
-          consecutiveFailures = 0;  // Reset on success
-        }
-      }
-      
-      // Handle coordinate-based actions (NEW - fallback when selectors fail) - ONLY if enabled
-      if (this.config.enableCoordinateMode && decision.coordinateAction && !iterationHadFailure) {
-        coordinateAttempts++;
-        
-        this.logger?.(`[Orchestrator] 🎯 Coordinate Action (attempt ${coordinateAttempts}/2): ${decision.coordinateAction.action} at (${decision.coordinateAction.xPercent}%, ${decision.coordinateAction.yPercent}%)`);
-        
-        try {
-          // BEFORE screenshot for visual verification
-          const beforeScreenshot = await page.screenshot({ encoding: 'base64', fullPage: false, type: 'jpeg', quality: 60 });
-          const beforeDataUrl = `data:image/jpeg;base64,${beforeScreenshot}`;
-          
-          // Generate Playwright commands from coordinate action
-          const coordCommands = await CoordinateConverter.generateCommands(decision.coordinateAction, page);
-          
-          this.logger?.(`[Orchestrator] Generated commands:`);
-          coordCommands.forEach(cmd => this.logger?.(`  ${cmd}`));
-          
-          // Execute coordinate commands
-          const coordResult = await this.executeCommands(
-            coordCommands,
-            page,
-            memory,
-            stepNumber,
-            iteration,
-            jobId
-          );
-          
-          commandsExecuted.push(...coordResult.executed);
-          
-          if (!coordResult.allSucceeded) {
-            this.logger?.(`[Orchestrator] ❌ Coordinate action failed (Playwright error)`);
-            consecutiveFailures++;
-            iterationHadFailure = true;
-            
-            // Give up after 2 coordinate attempts
-            if (coordinateAttempts >= 2) {
-              this.logger?.(`[Orchestrator] 🛑 Coordinate mode exhausted (2 attempts) - marking stuck`);
-              return {
-                success: false,
-                commands: commandsExecuted,
-                iterations: iteration,
-                terminationReason: 'agent_stuck',
-                memory,
-                error: 'Coordinate fallback failed after 2 attempts - unable to proceed'
-              };
-            }
-          } else {
-            this.logger?.(`[Orchestrator] ✅ Coordinate action succeeded (no Playwright error)`);
-            
-            // CRITICAL: Verify visually if goal was achieved (coordinates might have clicked wrong place)
-            // Wait for network idle (smarter than fixed timeout - waits as needed, max 10s)
-            try {
-              await page.waitForLoadState('networkidle', { timeout: 10000 });
-            } catch (e) {
-              // Network idle timeout - page may still be loading, but proceed with verification
-              this.logger?.(`[Orchestrator] ⚠️  Network idle timeout after 10s, proceeding with verification`, 'warn');
-            }
-            
-            const afterScreenshot = await page.screenshot({ encoding: 'base64', fullPage: false, type: 'jpeg', quality: 60 });
-            const afterDataUrl = `data:image/jpeg;base64,${afterScreenshot}`;
-            
-            this.logger?.(`[Orchestrator] 📸 Verifying coordinate action visually...`);
-            
-            const verificationRequest = {
-              model: 'gpt-5-mini',
-              systemPrompt: 'You are a visual verification expert for web automation. Compare before/after screenshots to determine if an action achieved its goal.',
-              userPrompt: `Goal: ${scenarioSteps[stepNumber - 1]}\n\nA coordinate-based action was just executed. Compare the BEFORE and AFTER screenshots.\n\nDid the action achieve the goal? Respond with JSON:\n{\n  "verified": boolean,\n  "reasoning": "What changed (or didn't change) between screenshots",\n  "visibleChanges": ["List of UI changes observed"],\n  "clickedWrongPlace": boolean\n}\n\nBe strict: Only return verified=true if you clearly see the expected change matching the goal.`,
-              images: [
-                { label: 'BEFORE', dataUrl: beforeDataUrl },
-                { label: 'AFTER', dataUrl: afterDataUrl }
-              ]
-            };
-            
-            const verificationResponse = await this.llmFacade.llmProvider.callLLM(verificationRequest);
-            const jsonMatch = verificationResponse.answer.match(/\{[\s\S]*\}/);
-            
-            if (jsonMatch) {
-              const verificationResult = JSON.parse(jsonMatch[0]);
-              const verified = verificationResult.verified === true;
-              const reasoning = verificationResult.reasoning || 'No reasoning provided';
-              const clickedWrongPlace = verificationResult.clickedWrongPlace === true;
-              
-              this.logger?.(`[Orchestrator] 📊 Visual verification: ${verified ? '✅ VERIFIED' : '❌ NOT VERIFIED'}`);
-              this.logger?.(`[Orchestrator] 💭 Reasoning: ${reasoning}`);
-              
-              if (verified) {
-                // Goal achieved! Can mark complete
-                consecutiveFailures = 0;
-                
-                // Store note for context
-                noteToSelf = {
-                  fromIteration: iteration,
-                  content: `Coordinate action verified successful: ${reasoning}`
-                };
-                memory.latestNote = noteToSelf;
-              } else {
-                // Goal NOT achieved despite no error
-                this.logger?.(`[Orchestrator] ⚠️  Coordinate click succeeded but goal NOT achieved`, 'warn');
-                consecutiveFailures++;
-                iterationHadFailure = true;
-                
-                // Store diagnostic note for next attempt
-                const diagnostic = clickedWrongPlace 
-                  ? `Clicked wrong place. ${reasoning}. Try different coordinates.`
-                  : `Action executed but goal not achieved. ${reasoning}. May need different approach.`;
-                
-                noteToSelf = {
-                  fromIteration: iteration,
-                  content: diagnostic
-                };
-                memory.latestNote = noteToSelf;
-                
-                // Give up after 2 coordinate attempts
-                if (coordinateAttempts >= 2) {
-                  this.logger?.(`[Orchestrator] 🛑 Coordinate mode exhausted (2 attempts, none achieved goal) - marking stuck`);
-                  return {
-                    success: false,
-                    commands: commandsExecuted,
-                    iterations: iteration,
-                    terminationReason: 'agent_stuck',
-                    memory,
-                    error: `Coordinate actions clicked but didn't achieve goal: ${reasoning}`
-                  };
-                }
-              }
-            } else {
-              this.logger?.(`[Orchestrator] ⚠️  Could not parse verification response - treating as unverified`, 'warn');
-              consecutiveFailures++;
-              iterationHadFailure = true;
-            }
-          }
-          
-        } catch (error: any) {
-          this.logger?.(`[Orchestrator] ❌ Coordinate action error: ${error.message}`, 'error');
-          consecutiveFailures++;
-          iterationHadFailure = true;
-          
-          // Give up after 2 coordinate attempts
-          if (coordinateAttempts >= 2) {
-            this.logger?.(`[Orchestrator] 🛑 Coordinate mode exhausted (2 attempts) - marking stuck`);
-            return {
-              success: false,
-              commands: commandsExecuted,
-              iterations: iteration,
-              terminationReason: 'agent_stuck',
-              memory,
-              error: 'Coordinate fallback failed after 2 attempts - unable to proceed'
-            };
-          }
-        }
-      }
-      
-      // System-enforced stuck detection (agent might not detect it)
-      // Allow 5 failures: 3 selector attempts + 2 coordinate attempts
-      if (consecutiveFailures >= 5) {
-        this.logger?.(`[Orchestrator] 🛑 SYSTEM: ${consecutiveFailures} consecutive failures detected - forcing stuck`, 'warn');
-        return {
-          success: false,
-          commands: commandsExecuted,
-          iterations: iteration,
-          terminationReason: 'agent_stuck',
-          memory,
-          error: `Failed ${consecutiveFailures} iterations in a row - unable to proceed`
-        };
-      }
-      
-      // Update memory with experiences
-      if (decision.experiences && decision.experiences.length > 0) {
-        for (const exp of decision.experiences) {
-          // Deduplicate - don't add if very similar experience exists
-          const exists = memory.experiences.some(existing =>
-            existing.toLowerCase().includes(exp.toLowerCase()) ||
-            exp.toLowerCase().includes(existing.toLowerCase())
-          );
-          
-          if (!exists) {
-            memory.experiences.push(exp);
-            this.logger?.(`[Orchestrator] 📚 Experience: ${exp}`);
-          }
-        }
-        
-        // Cap experiences
-        if (memory.experiences.length > this.config.maxExperiences) {
-          memory.experiences = memory.experiences.slice(-this.config.maxExperiences);
-        }
-      }
-      
-      // Store note to future self (tactical memory across iterations AND steps)
-      if (decision.noteToFutureSelf) {
-        noteToSelf = {
-          fromIteration: iteration,
-          content: decision.noteToFutureSelf
-        };
-        memory.latestNote = noteToSelf;  // Persist in journey memory across steps
-        this.logger?.(`[Orchestrator] 📝 Note to self: ${decision.noteToFutureSelf}`);
-      }
-      
-      // Check termination
-      if (decision.status !== 'continue') {
-        this.logger?.(`[Orchestrator] 🎯 Status: ${decision.status}`);
-        this.logger?.(`[Orchestrator] 💭 Reason: ${decision.statusReasoning}`);
-        
-        // SAFETY CHECK: Don't allow "complete" if commands failed this iteration
-        if (decision.status === 'complete' && iterationHadFailure) {
-          this.logger?.(`[Orchestrator] ⚠️  OVERRIDE: Agent said "complete" but commands FAILED - forcing "continue"`, 'warn');
-          this.logger?.(`[Orchestrator] 💭 System: Commands must succeed before marking complete`);
-          // Don't return - continue to next iteration
-        } else {
-          // Valid termination
-          return {
-            success: decision.status === 'complete',
-            commands: commandsExecuted,
-            iterations: iteration,
-            terminationReason: decision.status === 'complete' ? 'complete' :
-                             decision.status === 'stuck' ? 'agent_stuck' :
-                             'infeasible',
-            memory
-          };
-        }
-      }
-    }
-    
-    // Hit iteration limit
-    this.logger?.(`[Orchestrator] ⚠ Maximum iterations reached (${this.config.maxIterationsPerStep})`);
-    
-    return {
-      success: false,
-      commands: commandsExecuted,
-      iterations: iteration,
-      terminationReason: 'system_limit',
-      memory,
-      error: 'Maximum iterations reached'
-    };
-  }
-  
-  /**
-   * Build context for agent
-   */
-  private async buildAgentContext(
-    page: any,
-    currentStepGoal: string,
-    stepNumber: number,
-    totalSteps: number,
-    scenarioSteps: string[],
-    memory: JourneyMemory,
-    consecutiveFailures?: number,
-    noteFromPreviousIteration?: NoteToFutureSelf,
-    priorSteps?: string[],    // NEW: For repair mode
-    nextSteps?: string[]      // NEW: For repair mode
-  ): Promise<AgentContext> {
-    // Get fresh DOM
-    const currentPageInfo = await getEnhancedPageInfo(page);
-    const currentURL = page.url();
-    
-    // Get recent steps
-    const recentSteps = memory.history.slice(-this.config.recentStepsCount);
-    
-    // SoM integration: Update markers and capture screenshot with visual IDs
-    let somScreenshot: string | undefined = undefined;
-    let somElementMap: string | undefined = undefined;
-    if (this.config.useSoM && this.somHandler) {
-      try {
-        if (!this.somHandler) {
-          this.somHandler = new PageSoMHandler(page, this.logger);
-        } else {
-          this.somHandler.setPage(page);
-        }
-        
-        // Wait briefly for page stability (handles first iteration + safety net for fast SPAs)
-        try {
-          await page.waitForLoadState('domcontentloaded', { timeout: 5000 });
-        } catch (error: any) {
-          // Page already loaded or timeout - continue
-        }
-        
-        // Update SoM markers
-        await this.somHandler.updateSom();
-        
-        // Get screenshot WITH markers (viewport only - agent can scroll or use take_screenshot for full page)
-        somScreenshot = await this.somHandler.getScreenshot(true, false, 60);
-        
-        // Get element map for disambiguation
-        somElementMap = this.somHandler.getSomElementMap();
-        
-        this.logger?.(`[Orchestrator] SoM screenshot captured for agent decision-making`, 'log');
-      } catch (error: any) {
-        this.logger?.(`[Orchestrator] Failed to capture SoM screenshot: ${error.message}`, 'error');
-      }
-    }
-    
-    // Build context
-    const context = {
-      overallGoal: scenarioSteps.join('\n'),
-      currentStepGoal,
-      stepNumber,
-      totalSteps,
-      completedSteps: scenarioSteps.slice(0, stepNumber - 1),
-      remainingSteps: scenarioSteps.slice(stepNumber),
-      currentPageInfo,
-      currentURL,
-      recentSteps,
-      experiences: memory.experiences,
-      extractedData: memory.extractedData,
-      noteFromPreviousIteration,  // Pass tactical note from previous iteration
-      somScreenshot,  // SoM screenshot with visual markers (current)
-      somElementMap,  // SoM element details for disambiguation
-      priorSteps,     // NEW: Repair context (undefined for script gen)
-      nextSteps       // NEW: Repair context (undefined for script gen)
-    };
-    
-    // Save current screenshot as previous for next iteration (for tool access)
-    if (somScreenshot) {
-      this.previousSomScreenshot = somScreenshot;
-    }
-    
-    return context;
-  }
-  
-  /**
-   * Call agent to make decision
-   */
-  private async callAgent(
-    context: AgentContext,
-    jobId: string,
-    stepNumber: number,
-    iteration: number,
-    consecutiveFailures?: number
-  ): Promise<AgentDecision> {
-    // Detect if coordinate mode should be activated
-    // Phase 1: Only 2 tiers (selectors → coordinates), so activate after 3 failures
-    // Phase 2: Will have 3 tiers (selectors → index → coordinates), threshold will be 5
-    const useCoordinateMode = consecutiveFailures !== undefined && consecutiveFailures >= 3;
-    
-    // Build appropriate system prompt based on mode
-    const toolDescriptions = this.toolRegistry.generateToolDescriptions();
-    let systemPrompt: string;
-    
-    if (this.config.useSoM) {
-      // SoM mode: Use visual element identification
-      systemPrompt = OrchestratorPrompts.buildSomSystemPrompt(this.config.somRestrictCoordinates);
-    } else if (useCoordinateMode) {
-      // Coordinate mode: Fallback when selectors fail
-      systemPrompt = OrchestratorPrompts.buildCoordinateSystemPrompt();
-    } else {
-      // Standard mode: DOM-based selectors
-      systemPrompt = OrchestratorPrompts.buildSystemPrompt(toolDescriptions, this.config.enableCoordinateMode);
-    }
-    
-    const userPrompt = OrchestratorPrompts.buildUserPrompt(context, consecutiveFailures, this.config.enableCoordinateMode);
-    
-    // Log prompt lengths for monitoring
-    const systemLength = systemPrompt.length;
-    const userLength = userPrompt.length;
-    const totalLength = systemLength + userLength;
-    const estimatedTokens = Math.ceil(totalLength / 4); // Rough estimate: 4 chars per token
-    
-    this.logger?.(`[Orchestrator] 📊 Prompt lengths: system=${systemLength} chars, user=${userLength} chars, total=${totalLength} chars (~${estimatedTokens} tokens)`, 'log');
-    
-    try {
-      // Call LLM directly via provider
-      const llmRequest: any = {
-        model: DEFAULT_MODEL,
-        systemPrompt,
-        userPrompt
-      };
-      
-      // Include current SoM screenshot as image
-      if (context.somScreenshot) {
-        llmRequest.imageUrl = context.somScreenshot;
-        this.logger?.(`[Orchestrator] Including SoM screenshot in LLM request`, 'log');
-      }
-      
-      const response = await this.llmFacade.llmProvider.callLLM(llmRequest);
-      
-      // Report token usage
-      if (response.usage && this.progressReporter?.onTokensUsed) {
-        const tokenUsage: TokenUsage = {
-          jobId,
-          stepNumber,
-          iteration,
-          inputTokens: response.usage.inputTokens,
-          outputTokens: response.usage.outputTokens,
-          includesImage: false,
-          model: DEFAULT_MODEL,
-          timestamp: Date.now()
-        };
-        this.logger?.(`[Orchestrator] 💰 Reporting token usage: ${tokenUsage.inputTokens} + ${tokenUsage.outputTokens}`, 'log');
-        await this.progressReporter.onTokensUsed(tokenUsage);
-      } else if (!response.usage) {
-        this.logger?.(`[Orchestrator] ⚠ No usage data in LLM response`, 'warn');
-      }
-      
-      // Parse response
-      return this.decisionParser.parse(response.answer);
-      
-    } catch (error: any) {
-      this.logger?.(`[Orchestrator] ✗ Agent call failed: ${error.message}`, 'error');
-      
-      // Return fallback decision
-      return {
-        status: 'stuck',
-        statusReasoning: `Agent call failed: ${error.message}`,
-        reasoning: 'LLM call failed'
-      };
-    }
-  }
-  
-  /**
-   * Execute tools
-   */
-  private async executeTools(
-    toolCalls: any[],
-    page: any,
-    memory: JourneyMemory,
-    stepNumber: number,
-    refMap?: Map<string, any>
-  ): Promise<Record<string, any>> {
-    this.logger?.(`[Orchestrator] 🔧 Executing ${toolCalls.length} tool(s)`);
-    
-    const results: Record<string, any> = {};
-    const toolContext: ToolExecutionContext & { refMap?: Map<string, any>; previousSomScreenshot?: string; somHandler?: any } = {
-      page,
-      memory,
-      stepNumber,
-      logger: this.logger,
-      refMap,  // Pass refMap for interact_with_ref tool
-      previousSomScreenshot: this.previousSomScreenshot,  // For view_previous_screenshot tool
-      somHandler: this.somHandler  // For refresh_som_markers tool
-    };
-    
-    for (const toolCall of toolCalls.slice(0, this.config.maxToolCallsPerIteration)) {
-      this.logger?.(`[Orchestrator]   ▶ ${toolCall.name}(${JSON.stringify(toolCall.params).substring(0, 50)}...)`);
-      
-      const result = await this.toolRegistry.execute(toolCall, toolContext);
-      results[toolCall.name] = result;
-      
-      if (result.success) {
-        this.logger?.(`[Orchestrator]   ✓ ${toolCall.name} succeeded`);
-      } else {
-        this.logger?.(`[Orchestrator]   ✗ ${toolCall.name} failed: ${result.error}`, 'error');
-      }
-    }
-    
-    return results;
-  }
-  
-  /**
-   * Parse SomCommand from command object
-   */
-  private parseSomCommand(cmd: any): SomCommand | null {
-    if (typeof cmd === 'object' && cmd.action) {
-      // Valid if: has elementRef, OR has coord, OR is navigation action
-      const isNavigationAction = ['navigate', 'goBack', 'goForward', 'reload'].includes(cmd.action);
-      const hasValidTarget = cmd.elementRef || cmd.coord || isNavigationAction;
-      
-      if (hasValidTarget) {
-        return {
-          elementRef: cmd.elementRef,
-          coord: cmd.coord,
-          action: cmd.action,
-          value: cmd.value,
-          fromCoord: cmd.fromCoord,
-          toCoord: cmd.toCoord,
-          force: cmd.force,
-          scrollAmount: cmd.scrollAmount,
-          scrollDirection: cmd.scrollDirection,
-          button: cmd.button,
-          clickCount: cmd.clickCount,
-          modifiers: cmd.modifiers,
-          delay: cmd.delay,
-          timeout: cmd.timeout
-        };
-      }
-    }
-    return null;
-  }
-  
-  /**
-   * Execute commands (mix of ref and playwright commands)
-   */
-  private async executeCommands(
-    commands: string[] | any[],
-    page: any,
-    memory: JourneyMemory,
-    stepNumber: number,
-    iteration: number,
-    jobId: string
-  ): Promise<{ executed: string[]; allSucceeded: boolean }> {
-    this.logger?.(`[Orchestrator] 📝 Executing ${commands.length} command(s)`);
-    
-    const executed: string[] = [];
-    
-    if (commands.length === 0) {
-      return { executed: [], allSucceeded: true };
-    }
-    
-    // SoM mode: Execute commands through PageSoMHandler
-    if (this.config.useSoM && this.somHandler) {
-      this.logger?.(`[Orchestrator] Using SoM mode for command execution`, 'log');
-      
-      for (let i = 0; i < commands.length; i++) {
-        const cmd = commands[i];
-        
-        // Check if verification or action command
-        if (isSomVerification(cmd)) {
-          // Handle verification command
-          try {
-            const result = await this.somHandler.executeVerification(cmd);
-            
-            // Always add command to executed array (even if verification failed)
-            // Scripts should contain the expect even if it fails during generation
-            if (result.playwrightCommand) {
-              executed.push(result.playwrightCommand);
-            }
-            
-            if (result.success) {
-              this.logger?.(`[Orchestrator]   ✓ [${i + 1}/${commands.length}] Verification passed`, 'log');
-              
-              memory.history.push({
-                stepNumber,
-                iteration,
-                action: `Verification ${i + 1}/${commands.length}: ${cmd.verificationType}`,
-                code: result.playwrightCommand,
-                result: 'success',
-                observation: `Verified: ${cmd.description || cmd.expected}`,
-                url: page.url(),
-                timestamp: Date.now()
-              });
-            } else {
-              this.logger?.(`[Orchestrator]   ✗ [${i + 1}/${commands.length}] Verification failed (non-fatal): ${result.error}`, 'warn');
-              
-              memory.history.push({
-                stepNumber,
-                iteration,
-                action: `Verification ${i + 1}/${commands.length} - FAILED`,
-                code: result.playwrightCommand || JSON.stringify(cmd),
-                result: 'failure',
-                observation: `Failed: ${result.error}`,
-                error: result.error,
-                url: page.url(),
-                timestamp: Date.now()
-              });
-              
-              // Continue anyway - verification failures are non-blocking for script generation
-            }
-            
-            // Small delay between commands
-            if (i < commands.length - 1) {
-              await page.waitForTimeout(300);
-            }
-            
-          } catch (error: any) {
-            this.logger?.(`[Orchestrator]   ✗ [${i + 1}/${commands.length}] Verification exception: ${error.message}`, 'error');
-          }
-          
-        } else if (isSomCommand(cmd)) {
-          // Handle action command (existing logic)
-          const somCommand = cmd as SomCommand;
-          
-          try {
-            const result = await this.somHandler.runCommand(
-              somCommand,
-              this.config.somUseSomIdBasedCommands || false
-            );
-            
-            if (result.status === CommandRunStatus.SUCCESS && result.successAttempt) {
-              this.logger?.(`[Orchestrator]   ✓ [${i + 1}/${commands.length}] SoM action succeeded`, 'log');
-              executed.push(result.successAttempt.command!);
-              
-              memory.history.push({
-                stepNumber,
-                iteration,
-                action: `SoM Action ${i + 1}/${commands.length}: ${somCommand.action}`,
-                code: result.successAttempt.command!,
-                result: 'success',
-                observation: 'Executed successfully',
-                url: page.url(),
-                timestamp: Date.now()
-              });
-              
-              // Small delay for form validation/animations
-              if (i < commands.length - 1) {
-                await page.waitForTimeout(300);
-              }
-            } else {
-              this.logger?.(`[Orchestrator]   ✗ [${i + 1}/${commands.length}] SoM action failed: ${result.error}`, 'error');
-              
-              memory.history.push({
-                stepNumber,
-                iteration,
-                action: `SoM Action ${i + 1}/${commands.length}: ${somCommand.action} - FAILED`,
-                code: JSON.stringify(somCommand),
-                result: 'failure',
-                observation: `Failed: ${result.error}`,
-                error: result.error,
-                url: page.url(),
-                timestamp: Date.now()
-              });
-              
-              // Refresh SoM after batch (DOM may have changed)
-              if (this.somHandler && page) {
-                this.somHandler.setPage(page);
-                await this.somHandler.updateSom();
-              }
-              
-              return { executed, allSucceeded: false };
-            }
-          } catch (error: any) {
-            this.logger?.(`[Orchestrator]   ✗ [${i + 1}/${commands.length}] SoM action exception: ${error.message}`, 'error');
-            
-            memory.history.push({
-              stepNumber,
-              iteration,
-              action: `SoM Action ${i + 1}/${commands.length} - EXCEPTION`,
-              code: JSON.stringify(somCommand),
-              result: 'failure',
-              observation: `Exception: ${error.message}`,
-              error: error.message,
-              url: page.url(),
-              timestamp: Date.now()
-            });
-            
-            // Refresh SoM after batch (DOM may have changed)
-            if (this.somHandler && page) {
-              this.somHandler.setPage(page);
-              await this.somHandler.updateSom();
-            }
-            
-            return { executed, allSucceeded: false };
-          }
-        } else {
-          this.logger?.(`[Orchestrator]   ⚠ [${i + 1}/${commands.length}] Not a valid SoM command/verification, skipping`, 'warn');
-        }
-      }
-      
-      // Always wait for page to stabilize after command batch
-      // This handles both explicit navigation AND clicks that trigger navigation/SPA routes
-      try {
-        this.logger?.(`[Orchestrator] Waiting for page to stabilize...`, 'log');
-        // Use networkidle with short timeout to catch navigation without blocking on SPAs with continuous requests
-        await page.waitForLoadState('networkidle', { timeout: 3000 });
-        this.logger?.(`[Orchestrator] Page stabilized (networkidle)`, 'log');
-      } catch (error: any) {
-        // If networkidle times out, fall back to domcontentloaded
-        try {
-          await page.waitForLoadState('domcontentloaded', { timeout: 2000 });
-          this.logger?.(`[Orchestrator] Page loaded (domcontentloaded)`, 'log');
-        } catch (error2: any) {
-          this.logger?.(`[Orchestrator] Page load wait timeout (continuing anyway)`, 'warn');
-        }
-      }
-      
-      // Refresh SoM after batch (DOM may have changed and page is now stable)
-      if (this.somHandler && page) {
-        this.somHandler.setPage(page);
-        await this.somHandler.updateSom();
-      }
-      
-      return { executed, allSucceeded: true };
-    }
-    
-    // Standard mode: Execute all commands in sequence with small delay between them
-    // Delay helps with form validation, button enabling, and animations
-    const wrappedCode = (commands as string[]).map((cmd, i) => `
-// Command ${i + 1}/${commands.length}
-try {
-  ${cmd}
-  __results.push({ index: ${i}, success: true });
-  ${i < commands.length - 1 ? 'await page.waitForTimeout(300);' : ''} // Small delay for form validation/animations
-} catch (error) {
-  __results.push({ index: ${i}, success: false, error: error.message });
-  throw error;
-}`).join('\n');
-    
-    const fullCode = `const __results = []; ${wrappedCode} return __results;`;
-    
-    try {
-      const func = new Function('page', 'expect', 'return (async () => { ' + fullCode + ' })()');
-      const results = await func(page, (global as any).expect);
-      
-      for (let i = 0; i < commands.length; i++) {
-        const cmd = commands[i];
-        const result = results[i];
-        
-        if (result && result.success) {
-          this.logger?.(`[Orchestrator]   ✓ [${i + 1}/${commands.length}] Success`);
-          memory.history.push({
-            stepNumber,
-            iteration,
-            action: `Command ${i + 1}/${commands.length}`,
-            code: cmd,
-            result: 'success',
-            observation: 'Executed successfully',
-            url: page.url(),
-            timestamp: Date.now()
-          });
-          executed.push(cmd);
-        }
-      }
-      
-      if (memory.history.length > this.config.maxHistorySize) {
-        memory.history = memory.history.slice(-this.config.maxHistorySize);
-      }
-      
-      return { executed, allSucceeded: true };
-      
-    } catch (error: any) {
-      const errorMessage = error.message || String(error);
-      this.logger?.(`[Orchestrator] ❌ Command execution failed: ${errorMessage}`, 'error');
-      
-      memory.history.push({
-        stepNumber,
-        iteration,
-        action: `Command - FAILED`,
-        code: commands[executed.length] || '',
-        result: 'failure',
-        observation: `Failed: ${errorMessage}`,
-        error: errorMessage,
-        url: page.url(),
-        timestamp: Date.now()
-      });
-      
-      return { executed, allSucceeded: false };
-    }
-  }
-
-  /**
-   * Report step progress
-   */
-  private async reportStepProgress(
-    jobId: string,
-    stepNumber: number,
-    description: string,
-    decision: AgentDecision,
-    iteration: number
-  ): Promise<void> {
-    if (!this.progressReporter?.onStepProgress) return;
-    
-    await this.progressReporter.onStepProgress({
-      jobId,
-      stepNumber,
-      description,
-      status: decision.status === 'complete' ? StepExecutionStatus.SUCCESS :
-              decision.status === 'stuck' || decision.status === 'infeasible' ? StepExecutionStatus.FAILURE :
-              StepExecutionStatus.IN_PROGRESS,
-      code: decision.commands?.join('\n'),
-      // Include agent metadata for transparency
-      agentIteration: iteration,
-      agentReasoning: decision.reasoning,
-      agentSelfReflection: decision.selfReflection,
-      agentExperiences: decision.experiences,
-      agentToolsUsed: decision.toolCalls?.map(t => t.name),
-      agentStatus: decision.status
-    });
-  }
-  
-  /**
-   * Execute exploration mode - agent autonomously explores to achieve journey goal
-   * Fires onStepProgress callbacks for each autonomous action (transparent to caller)
-   */
-  async executeExploration(
-    page: any,
-    explorationConfig: ExplorationMode,
-    jobId: string
-  ): Promise<OrchestratorStepResult> {
-    this.logger?.(`\n[Orchestrator] ========== EXPLORATION MODE ==========`);
-    this.logger?.(`[Orchestrator] 🎯 Journey Goal: ${explorationConfig.explorationPrompt}`);
-    if (explorationConfig.testDataPrompt) {
-      this.logger?.(`[Orchestrator] 📋 Test Data: ${explorationConfig.testDataPrompt}`);
-    }
-    
-    const memory: JourneyMemory = {
-      history: [],
-      experiences: [],
-      extractedData: {}
-    };
-    
-    const maxSteps = explorationConfig.maxExplorationSteps || 50;
-    let stepNumber = 0;
-    const commandsExecuted: string[] = [];
-    
-    while (stepNumber < maxSteps) {
-      stepNumber++;
-      
-      this.logger?.(`\n[Orchestrator] === Exploration Step ${stepNumber}/${maxSteps} ===`);
-      
-      // Build exploratory context
-      const context = await this.buildExploratoryContext(
-        page,
-        explorationConfig.explorationPrompt,
-        explorationConfig.testDataPrompt,
-        memory,
-        stepNumber,
-        maxSteps
-      );
-      
-      // Call agent with exploratory prompt
-      const decision = await this.callExploratoryAgent(
-        context,
-        jobId,
-        stepNumber
-      );
-      
-      this.decisionParser.log(decision, stepNumber);
-      
-      // Report step start (fires JourneyRunner's beforeStepStart callback)
-      if (this.progressReporter?.onStepProgress) {
-        const stepInfo = {
-          jobId,
-          stepNumber,
-          stepId: `exploration-${stepNumber}-${Date.now()}`,
-          description: decision.reasoning,
-          code: '',  // Will be filled after commands execute
-          status: StepExecutionStatus.IN_PROGRESS,
-          wasRepaired: false
-        };
-        await this.progressReporter.onStepProgress(stepInfo);
-      }
-      
-      // Execute tools if requested
-      if (decision.toolCalls && decision.toolCalls.length > 0) {
-        const toolResults = await this.executeTools(decision.toolCalls, page, memory, stepNumber);
-        
-        // If needs tool results, call agent again
-        if (decision.needsToolResults) {
-          const updatedContext = { ...context, toolResults };
-          const continuedDecision = await this.callExploratoryAgent(updatedContext, jobId, stepNumber);
-          
-          decision.commands = continuedDecision.commands || decision.commands;
-          decision.commandReasoning = continuedDecision.commandReasoning || decision.commandReasoning;
-          decision.status = continuedDecision.status;
-        }
-      }
-      
-      // Handle blocker clearing
-      if (decision.blockerDetected && decision.blockerDetected.clearingCommands) {
-        this.logger?.(`[Orchestrator] 🚧 Clearing blocker: ${decision.blockerDetected.description}`);
-        const blockerResult = await this.executeCommands(
-          decision.blockerDetected.clearingCommands,
-          page,
-          memory,
-          stepNumber,
-          1,
-          jobId
-        );
-        commandsExecuted.push(...blockerResult.executed);
-      }
-      
-      // Execute exploration commands
-      let commandsSucceeded = true;
-      if (decision.commands && decision.commands.length > 0) {
-        const executeResult = await this.executeCommands(
-          decision.commands,
-          page,
-          memory,
-          stepNumber,
-          1,
-          jobId
-        );
-        commandsExecuted.push(...executeResult.executed);
-        commandsSucceeded = executeResult.allSucceeded;
-      }
-      
-      // Report step completion (fires JourneyRunner's onStepComplete callback)
-      if (this.progressReporter?.onStepProgress) {
-        const stepInfo = {
-          jobId,
-          stepNumber,
-          stepId: `exploration-${stepNumber}-${Date.now()}`,
-          description: decision.reasoning,
-          code: decision.commands?.join('\n') || '',
-          status: commandsSucceeded ? StepExecutionStatus.SUCCESS : StepExecutionStatus.FAILURE,
-          error: commandsSucceeded ? undefined : 'Command execution failed',
-          wasRepaired: false
-        };
-        await this.progressReporter.onStepProgress(stepInfo);
-      }
-      
-      // Add experiences (both app patterns AND exploration progress)
-      if (decision.experiences) {
-        memory.experiences.push(...decision.experiences);
-        if (memory.experiences.length > this.config.maxExperiences) {
-          memory.experiences = memory.experiences.slice(-this.config.maxExperiences);
-        }
-      }
-      
-      // Store note for next iteration
-      if (decision.noteToFutureSelf) {
-        memory.latestNote = {
-          fromIteration: stepNumber,
-          content: decision.noteToFutureSelf
-        };
-      }
-      
-      // Check termination
-      if (decision.status === 'complete') {
-        this.logger?.(`[Orchestrator] ✅ Journey exploration complete: ${decision.statusReasoning}`);
-        return {
-          success: true,
-          commands: commandsExecuted,
-          iterations: stepNumber,
-          terminationReason: 'complete',
-          memory
-        };
-      } else if (decision.status === 'stuck') {
-        this.logger?.(`[Orchestrator] ❌ Exploration stuck: ${decision.statusReasoning}`);
-        return {
-          success: false,
-          commands: commandsExecuted,
-          iterations: stepNumber,
-          terminationReason: 'agent_stuck',
-          memory,
-          error: decision.statusReasoning
-        };
-      }
-    }
-    
-    // Hit max steps - not necessarily a failure
-    this.logger?.(`[Orchestrator] ⚠ Maximum exploration steps reached (budget limit)`);
-    return {
-      success: true,  // Not a failure - just budget limit
-      commands: commandsExecuted,
-      iterations: stepNumber,
-      terminationReason: 'system_limit',
-      memory
-    };
-  }
-  
-  private async buildExploratoryContext(
-    page: any,
-    explorationPrompt: string,
-    testDataPrompt: string | undefined,
-    memory: JourneyMemory,
-    stepNumber: number,
-    maxSteps: number
-  ): Promise<AgentContext> {
-    // Wait for page to be ready and elements to appear (especially important after navigation)
-    const currentPageInfo = await PageInfoRetry.getWithRetry(page);
-    const currentURL = page.url();
-    const recentSteps = memory.history.slice(-this.config.recentStepsCount);
-    
-    // SoM integration for exploratory mode
-    let somScreenshot: string | undefined = undefined;
-    let somElementMap: string | undefined = undefined;
-    if (this.config.useSoM && this.somHandler) {
-      try {
-        this.somHandler.setPage(page);
-        
-        // Wait briefly for page stability (handles first iteration + safety net for fast SPAs)
-        try {
-          await page.waitForLoadState('domcontentloaded', { timeout: 2000 });
-        } catch (error: any) {
-          // Page already loaded or timeout - continue
-        }
-        
-        // Update SoM markers
-        await this.somHandler.updateSom();
-        somScreenshot = await this.somHandler.getScreenshot(true, false, 60);  // Viewport only - agent can scroll or request full page
-        
-        // Get element map for disambiguation
-        somElementMap = this.somHandler.getSomElementMap();
-        
-        this.logger?.(`[Orchestrator] SoM screenshot captured for exploratory agent`, 'log');
-      } catch (error: any) {
-        this.logger?.(`[Orchestrator] Failed to capture SoM screenshot: ${error.message}`, 'error');
-      }
-    }
-    
-    const context = {
-      overallGoal: explorationPrompt,
-      currentStepGoal: explorationPrompt,  // Same as overall for single journey
-      stepNumber,
-      totalSteps: maxSteps,
-      completedSteps: [],
-      remainingSteps: [],
-      currentPageInfo,
-      currentURL,
-      recentSteps,
-      experiences: memory.experiences,
-      extractedData: memory.extractedData,
-      noteFromPreviousIteration: memory.latestNote,
-      testDataPrompt,  // CRITICAL: Store testDataPrompt in context
-      somScreenshot,  // SoM screenshot for exploratory mode (current)
-      somElementMap   // SoM element details for disambiguation
-    };
-    
-    // Save current screenshot as previous for next iteration (for tool access)
-    if (somScreenshot) {
-      this.previousSomScreenshot = somScreenshot;
-    }
-    
-    return context;
-  }
-  
-  private async callExploratoryAgent(
-    context: AgentContext,
-    jobId: string,
-    stepNumber: number
-  ): Promise<AgentDecision> {
-    const toolDescriptions = this.toolRegistry.generateToolDescriptions();
-    
-    // Use SoM system prompt if in SoM mode, otherwise use standard exploratory prompt
-    const systemPrompt = this.config.useSoM
-      ? OrchestratorPrompts.buildSomSystemPrompt(this.config.somRestrictCoordinates)
-      : OrchestratorPrompts.buildExploratorySystemPrompt(toolDescriptions);
-      
-    const userPrompt = OrchestratorPrompts.buildExploratoryUserPrompt(
-      context,
-      context.overallGoal,
-      context.testDataPrompt,  // Pass testDataPrompt from context
-      stepNumber,
-      context.totalSteps
-    );
-    
-    const llmRequest: any = {
-      model: DEFAULT_MODEL,
-      systemPrompt,
-      userPrompt
-    };
-    
-    // Include current SoM screenshot as image
-    if (context.somScreenshot) {
-      llmRequest.imageUrl = context.somScreenshot;
-      this.logger?.(`[Orchestrator] Including SoM screenshot in exploratory LLM request`, 'log');
-    }
-    
-    const response = await this.llmFacade.llmProvider.callLLM(llmRequest);
-    
-    // Report token usage
-    if (response.usage && this.progressReporter?.onTokensUsed) {
-      await this.progressReporter.onTokensUsed({
-        jobId,
-        stepNumber,
-        iteration: 1,
-        inputTokens: response.usage.inputTokens,
-        outputTokens: response.usage.outputTokens,
-        includesImage: false,
-        model: DEFAULT_MODEL,
-        timestamp: Date.now()
-      });
-    }
-    
-    // Parse response (same JSON format as regular mode)
-    const decision = this.decisionParser.parse(response.answer);
-    return decision;
-  }
-}
-
-