testchimp-runner-core 0.0.21 → 0.0.23

package/src/orchestrator/orchestrator-agent.ts

@@ -0,0 +1,1037 @@
+/**
+ * Orchestrator Agent
+ * Single agent with tool-use capabilities that maintains journey memory and executes scenarios
+ */
+
+import { expect } from '@playwright/test';
+import { LLMFacade } from '../llm-facade';
+import { ProgressReporter, TokenUsage, StepExecutionStatus } from '../progress-reporter';
+import { getEnhancedPageInfo } from '../utils/page-info-utils';
+import { ToolRegistry, ToolExecutionContext } from './tool-registry';
+import { DEFAULT_MODEL } from '../model-constants';
+import {
+  AgentConfig,
+  AgentContext,
+  AgentDecision,
+  JourneyMemory,
+  MemoryStep,
+  OrchestratorStepResult,
+  SelfReflection,
+  DEFAULT_AGENT_CONFIG
+} from './types';
+
+/**
+ * Orchestrator Agent - manages step execution with tool use and memory
+ */
+export class OrchestratorAgent {
+  private llmFacade: LLMFacade;
+  private toolRegistry: ToolRegistry;
+  private progressReporter?: ProgressReporter;
+  private config: Required<AgentConfig>;
+  private logger?: (message: string, level?: 'log' | 'error' | 'warn' | 'debug') => void;
+  private debugMode: boolean = false;
+  
+  constructor(
+    llmFacade: LLMFacade,
+    toolRegistry: ToolRegistry,
+    config?: Partial<AgentConfig>,
+    progressReporter?: ProgressReporter,
+    logger?: (message: string, level?: 'log' | 'error' | 'warn' | 'debug') => void,
+    debugMode?: boolean
+  ) {
+    this.llmFacade = llmFacade;
+    this.toolRegistry = toolRegistry;
+    this.config = { ...DEFAULT_AGENT_CONFIG, ...config };
+    this.progressReporter = progressReporter;
+    this.logger = logger;
+    this.debugMode = debugMode || false;
+  }
+  
+  setDebugMode(enabled: boolean): void {
+    this.debugMode = enabled;
+  }
+  
+  /**
+   * Execute a single step of the scenario
+   */
+  async executeStep(
+    page: any,
+    stepDescription: string,
+    stepNumber: number,
+    totalSteps: number,
+    scenarioSteps: string[],
+    memory: JourneyMemory,
+    jobId: string
+  ): Promise<OrchestratorStepResult> {
+    this.logger?.(`\n[Orchestrator] ========== STEP ${stepNumber}/${totalSteps} ==========`);
+    this.logger?.(`[Orchestrator] 🎯 Goal: ${stepDescription}`);
+    
+    let iteration = 0;
+    let previousReflection: SelfReflection | undefined = undefined;
+    const commandsExecuted: string[] = [];
+    let consecutiveFailures = 0;  // Track consecutive iterations with failed commands
+    
+    while (iteration < this.config.maxIterationsPerStep) {
+      iteration++;
+      
+      this.logger?.(`\n[Orchestrator] === Iteration ${iteration}/${this.config.maxIterationsPerStep} ===`);
+      
+      // Build context for agent
+      const context = await this.buildAgentContext(
+        page,
+        stepDescription,
+        stepNumber,
+        totalSteps,
+        scenarioSteps,
+        memory,
+        previousReflection,
+        consecutiveFailures
+      );
+      
+      // Call agent to make decision
+      const decision = await this.callAgent(context, jobId, stepNumber, iteration, consecutiveFailures);
+      
+      // Log agent's reasoning
+      this.logAgentDecision(decision, iteration);
+      
+      // Report progress
+      await this.reportStepProgress(jobId, stepNumber, stepDescription, decision, iteration);
+      
+      // Execute tools if requested (tools are READ-ONLY, they don't change state)
+      let toolResults: Record<string, any> = {};
+      
+      if (decision.toolCalls && decision.toolCalls.length > 0) {
+        toolResults = await this.executeTools(decision.toolCalls, page, memory, stepNumber);
+        
+        // If agent wants to wait for tool results before proceeding, call agent again
+        if (decision.needsToolResults) {
+          const updatedContext = { ...context, toolResults };
+          const continuedDecision = await this.callAgent(updatedContext, jobId, stepNumber, iteration, consecutiveFailures);
+          
+          // Merge continued decision
+          decision.commands = continuedDecision.commands || decision.commands;
+          decision.commandReasoning = continuedDecision.commandReasoning || decision.commandReasoning;
+          decision.status = continuedDecision.status;
+          decision.statusReasoning = continuedDecision.statusReasoning;
+          decision.reasoning = continuedDecision.reasoning;
+        }
+      }
+      
+      // Execute commands sequentially
+      let iterationHadFailure = false;
+      
+      // Handle blocker if detected (clear blocker FIRST, then proceed with main commands)
+      if (decision.blockerDetected && decision.blockerDetected.clearingCommands && decision.blockerDetected.clearingCommands.length > 0) {
+        this.logger?.(`[Orchestrator] 🚧 BLOCKER DETECTED: ${decision.blockerDetected.description}`);
+        this.logger?.(`[Orchestrator] 🧹 Clearing blocker with ${decision.blockerDetected.clearingCommands.length} command(s)...`);
+        
+        const blockerResult = await this.executeCommandsSequentially(
+          decision.blockerDetected.clearingCommands,
+          page,
+          memory,
+          stepNumber,
+          iteration,
+          jobId
+        );
+        
+        // Add blocker commands with comment to output
+        if (blockerResult.executed.length > 0) {
+          commandsExecuted.push(`// Blocker: ${decision.blockerDetected.description}`);
+          commandsExecuted.push(...blockerResult.executed);
+        }
+        
+        // If blocker clearing failed, track it
+        if (!blockerResult.allSucceeded) {
+          this.logger?.(`[Orchestrator] ❌ Failed to clear blocker - continuing anyway`);
+          consecutiveFailures++;
+          iterationHadFailure = true;
+        }
+      }
+      
+      // Execute main commands (only if no blocker failure or continueOnFailure is true)
+      if (decision.commands && decision.commands.length > 0 && !iterationHadFailure) {
+        const executeResult = await this.executeCommandsSequentially(
+          decision.commands,
+          page,
+          memory,
+          stepNumber,
+          iteration,
+          jobId
+        );
+        
+        commandsExecuted.push(...executeResult.executed);
+        
+        // Track failures
+        if (!executeResult.allSucceeded) {
+          this.logger?.(`[Orchestrator] ⚠ Command execution stopped at failure`);
+          consecutiveFailures++;
+          iterationHadFailure = true;
+        } else {
+          consecutiveFailures = 0;  // Reset on success
+        }
+      }
+      
+      // System-enforced stuck detection (agent might not detect it)
+      if (consecutiveFailures >= 3) {
+        this.logger?.(`[Orchestrator] 🛑 SYSTEM: ${consecutiveFailures} consecutive failures detected - forcing stuck`, 'warn');
+        return {
+          success: false,
+          commands: commandsExecuted,
+          iterations: iteration,
+          terminationReason: 'agent_stuck',
+          memory,
+          error: `Failed ${consecutiveFailures} iterations in a row - unable to proceed`
+        };
+      }
+      
+      // Update memory with experiences
+      if (decision.experiences && decision.experiences.length > 0) {
+        for (const exp of decision.experiences) {
+          // Deduplicate - don't add if very similar experience exists
+          const exists = memory.experiences.some(existing =>
+            existing.toLowerCase().includes(exp.toLowerCase()) ||
+            exp.toLowerCase().includes(existing.toLowerCase())
+          );
+          
+          if (!exists) {
+            memory.experiences.push(exp);
+            this.logger?.(`[Orchestrator] 📚 Experience: ${exp}`);
+          }
+        }
+        
+        // Cap experiences
+        if (memory.experiences.length > this.config.maxExperiences) {
+          memory.experiences = memory.experiences.slice(-this.config.maxExperiences);
+        }
+      }
+      
+      // Store self-reflection for next iteration
+      previousReflection = decision.selfReflection;
+      
+      // Check termination
+      if (decision.status !== 'continue') {
+        this.logger?.(`[Orchestrator] 🎯 Status: ${decision.status}`);
+        this.logger?.(`[Orchestrator] 💭 Reason: ${decision.statusReasoning}`);
+        
+        return {
+          success: decision.status === 'complete',
+          commands: commandsExecuted,
+          iterations: iteration,
+          terminationReason: decision.status === 'complete' ? 'complete' :
+                           decision.status === 'stuck' ? 'agent_stuck' :
+                           'infeasible',
+          memory
+        };
+      }
+    }
+    
+    // Hit iteration limit
+    this.logger?.(`[Orchestrator] ⚠ Maximum iterations reached (${this.config.maxIterationsPerStep})`);
+    
+    return {
+      success: false,
+      commands: commandsExecuted,
+      iterations: iteration,
+      terminationReason: 'system_limit',
+      memory,
+      error: 'Maximum iterations reached'
+    };
+  }
+  
+  /**
+   * Build context for agent
+   */
+  private async buildAgentContext(
+    page: any,
+    currentStepGoal: string,
+    stepNumber: number,
+    totalSteps: number,
+    scenarioSteps: string[],
+    memory: JourneyMemory,
+    previousReflection?: SelfReflection,
+    consecutiveFailures?: number
+  ): Promise<AgentContext> {
+    // Get fresh DOM
+    const currentPageInfo = await getEnhancedPageInfo(page);
+    const currentURL = page.url();
+    
+    // Get recent steps
+    const recentSteps = memory.history.slice(-this.config.recentStepsCount);
+    
+    // Build context
+    return {
+      overallGoal: scenarioSteps.join('\n'),
+      currentStepGoal,
+      stepNumber,
+      totalSteps,
+      completedSteps: scenarioSteps.slice(0, stepNumber - 1),
+      remainingSteps: scenarioSteps.slice(stepNumber),
+      currentPageInfo,
+      currentURL,
+      recentSteps,
+      experiences: memory.experiences,
+      extractedData: memory.extractedData,
+      previousIterationGuidance: previousReflection
+    };
+  }
+  
+  /**
+   * Call agent to make decision
+   */
+  private async callAgent(
+    context: AgentContext,
+    jobId: string,
+    stepNumber: number,
+    iteration: number,
+    consecutiveFailures?: number
+  ): Promise<AgentDecision> {
+    // Build prompt
+    const systemPrompt = this.buildSystemPrompt();
+    const userPrompt = this.buildUserPrompt(context, consecutiveFailures);
+    
+    try {
+      // Call LLM directly via provider
+      const llmRequest = {
+        model: DEFAULT_MODEL,
+        systemPrompt,
+        userPrompt
+      };
+      
+      const response = await this.llmFacade.llmProvider.callLLM(llmRequest);
+      
+      // Report token usage
+      if (response.usage && this.progressReporter?.onTokensUsed) {
+        const tokenUsage: TokenUsage = {
+          jobId,
+          stepNumber,
+          iteration,
+          inputTokens: response.usage.inputTokens,
+          outputTokens: response.usage.outputTokens,
+          includesImage: false,
+          model: DEFAULT_MODEL,
+          timestamp: Date.now()
+        };
+        this.logger?.(`[Orchestrator] 💰 Reporting token usage: ${tokenUsage.inputTokens} + ${tokenUsage.outputTokens}`, 'log');
+        await this.progressReporter.onTokensUsed(tokenUsage);
+      } else if (!response.usage) {
+        this.logger?.(`[Orchestrator] ⚠ No usage data in LLM response`, 'warn');
+      }
+      
+      // Parse response
+      return this.parseAgentDecision(response.answer);
+      
+    } catch (error: any) {
+      this.logger?.(`[Orchestrator] ✗ Agent call failed: ${error.message}`, 'error');
+      
+      // Return fallback decision
+      return {
+        status: 'stuck',
+        statusReasoning: `Agent call failed: ${error.message}`,
+        reasoning: 'LLM call failed'
+      };
+    }
+  }
+  
+  /**
+   * Build system prompt with tool descriptions
+   */
+  private buildSystemPrompt(): string {
+    const toolDescriptions = this.toolRegistry.generateToolDescriptions();
+    
+    return `You are an intelligent test automation agent that executes web scenarios using Playwright.
+
+${toolDescriptions}
+
+YOUR RESPONSE FORMAT (JSON):
+{
+  "toolCalls": [{"name": "tool_name", "params": {...}}],  // Optional: Tools to use
+  "toolReasoning": "Why I need these tools",              // If using tools
+  "needsToolResults": true/false,                         // Wait for tool results before commands?
+  "commands": ["await page.click(...)", ...],             // Playwright commands to execute
+  "commandReasoning": "Why these commands",               // If executing commands
+  "selfReflection": {                                     // Guidance for your next iteration
+    "guidanceForNext": "What to focus on next",
+    "detectingLoop": false,                               // Set true if repeating same approach
+    "loopReasoning": "Why I think I'm looping"           // If detecting loop
+  },
+  "experiences": ["App-specific learning"],              // APP-SPECIFIC patterns (see rules below)
+  "status": "continue|complete|stuck|infeasible",        // Your decision
+  "statusReasoning": "Why this status",
+  "reasoning": "Overall reasoning for this iteration",
+  "blockerDetected": {                                   // OPTIONAL: If something blocks the current goal
+    "description": "What's blocking",                    // E.g., "Cookie consent modal"
+    "clearingCommands": ["await page.click(...)"]       // Commands to clear blocker FIRST
+  },
+  "stepReEvaluation": {                                  // OPTIONAL: If questioning step order (last resort)
+    "detected": true,                                    // Set true if you think step order is wrong
+    "issue": "prior_incomplete|already_done|wrong_order", // What's the issue?
+    "explanation": "Why step order seems wrong"          // Your reasoning
+  }
+}
+
+STATUS DECISION RULES (CRITICAL!):
+
+- "complete": ALL commands succeeded AND step goal achieved
+  ❌ NEVER mark complete if ANY command failed!
+  
+- "continue": Command failed OR need more actions
+  Use when: Need to retry with different selector, wait for something, or continue execution
+  
+- "stuck": Tried 3+ iterations, all failed
+  
+- "infeasible": Goal truly impossible (element doesn't exist anywhere)
+
+IF COMMAND FAILED → status must be "continue" (to retry) or "stuck" (if tried many times)
+IF COMMAND SUCCEEDED → status can be "complete" (if goal done) or "continue" (if more needed)
+
+STEP RE-EVALUATION (LAST RESORT - After 2+ Failures):
+
+When you've failed multiple times on current step, QUESTION YOUR ASSUMPTIONS:
+
+1. **LOOK BACK**: Did prior steps really complete?
+   - Check COMPLETED steps list vs current page state
+   - Example: Says "Login complete" but you see login form → PRIOR STEP FAILED
+   - If prior step incomplete: Execute commands to complete it, explain in reasoning
+
+2. **LOOK FORWARD**: Is current step already done or no longer relevant?
+   - Check if current step goal already achieved (element exists, state correct)
+   - Example: Step says "Click Submit" but form already submitted → STEP ALREADY DONE
+   - If already done: Mark "complete" and explain why
+
+3. **LOOK AHEAD**: Is next step more feasible?
+   - Sometimes UI flows differently than expected
+   - Example: Current step "Click Edit" fails, but next step "Fill Name" elements are visible
+   - If can do future step: Note in reasoning, but PREFER sticking to order
+
+STICK TO ORIGINAL PLAN unless clear evidence suggests otherwise.
+Only re-evaluate after repeated failures on same step.
+
+BLOCKER DETECTION (IMPORTANT!):
+
+Use "blockerDetected" when the CURRENT STEP GOAL is not immediately feasible because something unexpected is in the way:
+
+Examples of BLOCKERS:
+- Cookie consent modal appears before login
+- "What's New" tour overlay prevents clicking navigation
+- Email verification prompt blocks form submission
+- Session timeout modal requires re-login before continuing
+
+When you detect a blocker:
+1. Set "blockerDetected.description" to describe what's blocking
+2. Provide "clearingCommands" to dismiss/handle the blocker
+3. Set your main "commands" to execute AFTER blocker is cleared
+4. Agent will execute: clearingCommands FIRST, then your commands
+
+Example:
+{
+  "blockerDetected": {
+    "description": "Cookie consent modal blocking page interaction",
+    "clearingCommands": ["await page.getByRole('button', {name: 'Accept'}).click()"]
+  },
+  "commands": ["await page.getByRole('textbox', {name: 'Email'}).fill('user@example.com')"],
+  ...
+}
+
+DON'T use blockerDetected for:
+- Normal step execution (just use commands)
+- Wrong selectors (that's not a blocker, just fix the selector)
+- Missing elements (that's infeasible, not blocked)
+
+EXPERIENCES - APP-SPECIFIC PATTERNS ONLY:
+
+Guidelines:
+- Concise - combine related info into single learning
+- Focus on app quirks, not obvious practices
+- Max 1-2 experiences per iteration
+- Capture SEMANTIC SELECTOR patterns when discovered
+
+✅ GOOD - App-specific patterns:
+- "Login form fields accessible via getByLabel: 'Email' and 'Password'"
+- "Submit buttons consistently use role=button with text matching action"
+- "Form uses custom dropdowns (role=combobox) instead of native selects"
+- "Navigation items accessible via getByRole('link') with visible text"
+- "Site uses data-testid attributes consistently (pattern: action-element-name)"
+- "Input fields have clear placeholders - prefer getByPlaceholder over IDs"
+
+❌ BAD - Too specific or obvious:
+- Listing individual field selectors (combine them)
+- "Button exists with role=button" (obvious)
+- Generic automation advice
+- Noting auto-generated IDs like #«r3»-form-item (these are unreliable)
+
+COMBINE related learnings instead of listing each element separately.
+PRIORITIZE learning about semantic selector availability over CSS selectors.
+
+CRITICAL RULES FOR DECISION MAKING:
+
+1. **BE FOCUSED AND CONCISE** (CRITICAL!):
+   - Read the CURRENT STEP GOAL carefully - ONLY do what's explicitly asked for in THAT step
+   - DON'T add extra actions, verifications, or defensive checks not mentioned in the step goal
+   - If step says "Send a message" → ONLY send the message, DON'T verify it appeared
+   - If step says "Click Submit" → ONLY click, DON'T wait or verify submission success
+   - ONLY add expect() assertions if the step explicitly mentions "verify", "check", "ensure", or "confirm"
+   - Keep commands minimal - no unnecessary preparatory or validation actions
+
+2. **COMMAND EFFICIENCY**:
+   - DON'T click on input field before filling it - just use .fill() directly
+   - DON'T do preparatory actions unless necessary
+   - Example: page.fill works without clicking the input first
+
+2b. **VARIABLE SCOPE & AVAILABLE FUNCTIONS** (CRITICAL!):
+   - Commands in SAME BATCH share scope - variables persist within iteration
+   - To save data for later steps: Use extract_data tool, then access via extractedData in context
+   - Within same batch: Variables work across commands
+   - AVAILABLE IN CONTEXT: page (Playwright Page object) and expect (Playwright assertions)
+   - You can use expect() directly - it is already imported and available
+
+3. **WHEN COMMAND FAILS** (most important!):
+   - DON'T repeat the same command/selector
+   - DO analyze the error message
+   - DO look at DOM snapshot for SEMANTIC selectors (getByRole, getByLabel, getByText)
+   - AVOID falling back to auto-generated IDs with unicode characters (e.g., #«r3»-form-item)
+   - DO take screenshot if DOM doesn't help
+   - "Element is not a <select>" → It's a custom dropdown, use .click() then select from menu
+   - "Timeout waiting for locator" → Try semantic selectors from ARIA tree instead of CSS IDs
+
+4. **CHECK YOUR RECENT STEPS FOR PATTERNS**:
+   - If you see same selector failed 2+ times → YOU ARE LOOPING!
+   - Set detectingLoop: true and explain why
+   - Take screenshot to see what's actually on the page
+   - Use different SEMANTIC selector from DOM/ARIA tree (getByRole, getByLabel, getByText, data-testid)
+   - AVOID repeating with similar auto-generated ID patterns (e.g., #«r3»-form-item → #«r4»-form-item)
+
+5. **USE DOM SNAPSHOT** - ONLY USE WHAT'S ACTUALLY THERE (CRITICAL!):
+   - You get INTERACTIVE ELEMENTS list showing: position, tag, id, text, SUGGESTED SELECTORS
+   - ONLY use selectors from this list - DO NOT invent selectors with text/names not in the list!
+   - ❌ WRONG: List shows button "Submit" but you use getByRole('button', {name: 'SomeOtherText'})
+   - ✅ RIGHT: Use EXACT selector from the provided list
+   - ✅ RIGHT: If list shows "#submit-btn" → use that
+   - ✅ RIGHT: If list shows "getByRole('button', {name: 'X'})" → use exactly that
+   - If element you need is NOT in list → take_screenshot to find it
+   - ARIA tree shows roles/names that exist - inventing roles/names not in tree will fail!
+
+5b. **SELECTOR PREFERENCE** (CRITICAL - Playwright Best Practices):
+   ALWAYS prefer user-friendly, semantic selectors in this order:
+   
+   🏆 BEST (use these first):
+   1. page.getByRole('role', {name: 'text'})     → ✅ Accessible, semantic, resilient
+   2. page.getByLabel('label text')              → ✅ Great for form inputs
+   3. page.getByPlaceholder('placeholder')       → ✅ Good for inputs without labels
+   4. page.getByText('visible text')             → ✅ Clear and readable
+   5. page.getByTestId('test-id')                → ✅ Stable if available
+   
+   ⚠️ AVOID (last resort only):
+   - CSS selectors with auto-generated IDs: #r3-form-item, #«r3»-form-item
+   - CSS selectors with unicode: #«r4»-form-item
+   - Complex CSS paths: div.ant-form > div:nth-child(2)
+   
+   Examples:
+   ❌ BAD:  await page.fill('#«r3»-form-item', 'alice@example.com')
+   ✅ GOOD: await page.getByLabel('Email').fill('alice@example.com')
+   ✅ GOOD: await page.getByRole('textbox', {name: 'Email'}).fill('alice@example.com')
+   ✅ GOOD: await page.getByPlaceholder('Enter your email').fill('alice@example.com')
+   
+   ❌ BAD:  await page.click('#submit-btn-23847')
+   ✅ GOOD: await page.getByRole('button', {name: 'Submit'}).click()
+   ✅ GOOD: await page.getByText('Submit').click()
+   
+   WHY: Semantic selectors are more maintainable, self-documenting, and resilient to UI changes.
+   Auto-generated IDs (especially with unicode like «r3») break when component instances change.
+
+5c. **ASSERTIONS AND VERIFICATION** (CRITICAL - Use expect):
+   Use expect() for verification ONLY when the step explicitly asks for verification.
+   
+   🏆 WHEN TO USE expect():
+   - Step says "Verify message appears" → await expect(locator).toBeVisible()
+   - Step says "Check field is empty" → await expect(locator).toBeEmpty()
+   - Step says "Confirm URL changed" → await expect(page).toHaveURL('...')
+   - Step explicitly mentions: "verify", "check", "ensure", "confirm", "validate"
+   
+   ❌ WHEN NOT TO USE expect():
+   - Step says "Send a message" → DON'T add expect() for message visibility
+   - Step says "Click Submit" → DON'T add expect() for button state or result
+   - Step says "Fill Email field" → DON'T add expect() to verify it's filled
+   - Step says "Navigate to Dashboard" → DON'T add expect() for URL (unless step asks)
+   
+   🏆 expect() syntax (when needed):
+   - await expect(locator).toBeVisible()
+   - await expect(locator).toHaveText('expected text')
+   - await expect(locator).toHaveValue('expected value')
+   - await expect(locator).toBeEmpty()
+   - await expect(page).toHaveURL('expected-url')
+   
+   ❌ BAD - Manual checks:
+   const value = await page.getByPlaceholder('Message...').inputValue();
+   if (value !== '') throw new Error(\`Expected empty but found: '\${value}'\`);
+   
+   ✅ GOOD - Use expect (but ONLY if step asks to verify):
+   await expect(page.getByPlaceholder('Message...')).toBeEmpty();
+   
+   WHY: expect() has built-in retries and better error messages.
+   BUT: Only use it when verification is explicitly requested in the step goal.
+
+6. **TOOLS ARE READ-ONLY (Information Gathering)**:
+   AVAILABLE TOOLS (for information only):
+   - take_screenshot: Get visual context when DOM isn't enough or stuck 2+ times
+     * DEFAULT: Captures FULL PAGE (elements below fold included)
+     * Set isFullPage=false ONLY if you need just viewport
+     * Use for: Planning, finding hidden elements, verifying layout
+   - recall_history: Remember earlier steps in journey
+   - inspect_page: Force fresh DOM snapshot (rarely needed, already in context)
+   - check_page_ready: Verify page loaded (use sparingly!)
+   - extract_data: Save data from page to memory for later steps
+   
+   ALL STATE CHANGES USE PLAYWRIGHT COMMANDS:
+   - Navigation: await page.goto('url')
+   - Go back: await page.goBack()
+   - Refresh: await page.reload()
+   - Fill inputs: await page.fill() or page.getByRole().fill()
+   - Click buttons: await page.click() or page.getByRole().click()
+   - Wait: await page.waitForURL(), page.waitForLoadState()
+   
+   Tools gather info → Commands change state
+
+7. **ERROR-SPECIFIC RESPONSES**:
+   - "Timeout waiting for locator" → Selector doesn't exist, try different one from DOM
+   - "Element is not a <select>" → It's a custom dropdown (button with role=combobox), use .click()
+   - "Multiple elements" → Add more specific selector from DOM (use #id)
+
+8. **WHEN TO RUN COMMANDS vs TOOLS**:
+   - Confident about selectors from DOM → Run commands directly
+   - Unsure or failed 2+ times → Take screenshot first
+   - First iteration of a step → Usually can run commands from DOM`;
+  }
+  
+  /**
+   * Build user prompt with context
+   */
+  private buildUserPrompt(context: AgentContext, consecutiveFailures?: number): string {
+    const parts: string[] = [];
+    
+    parts.push('=== YOUR CURRENT CONTEXT ===\n');
+    
+    // System warnings for accumulated failures
+    if (consecutiveFailures && consecutiveFailures >= 2) {
+      parts.push(`⚠️⚠️⚠️ SYSTEM WARNING ⚠️⚠️⚠️`);
+      parts.push(`You've failed ${consecutiveFailures} iterations in a row!`);
+      parts.push(`Next failure will force STUCK. You MUST:`);
+      parts.push(`1. Take screenshot if you haven't already`);
+      parts.push(`2. Try completely different selector strategy`);
+      parts.push(`3. QUESTION ASSUMPTIONS: Am I at the right step?`);
+      parts.push(`   - LOOK BACK: Did prior steps really complete? Check page state vs COMPLETED list`);
+      parts.push(`   - LOOK FORWARD: Is current step already done? Check if goal already achieved`);
+      parts.push(`4. OR mark status="stuck" if truly can't proceed`);
+      parts.push(`⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️⚠️\n`);
+    }
+    
+    // Goals - make current step very prominent
+    parts.push(`🎯 CURRENT STEP GOAL (${context.stepNumber}/${context.totalSteps}):`);
+    parts.push(`${context.currentStepGoal}`);
+    parts.push(`^^ DO ONLY WHAT THIS STEP ASKS - NO EXTRA ACTIONS OR VERIFICATIONS ^^`);
+    parts.push(`^^ If step doesn't say "verify/check/confirm" → DON'T add expect() assertions ^^`);
+    parts.push(`^^ Mark "complete" ONLY if commands succeeded ^^`);
+    parts.push(``);
+    parts.push(`OVERALL SCENARIO: ${context.overallGoal}\n`);
+    
+    if (context.completedSteps.length > 0) {
+      parts.push(`COMPLETED: ${context.completedSteps.join(', ')}`);
+    }
+    if (context.remainingSteps.length > 0) {
+      parts.push(`REMAINING: ${context.remainingSteps.join(', ')}\n`);
+    }
+    
+    // Current page state
+    parts.push(`\nCURRENT PAGE:`);
+    parts.push(`URL: ${context.currentURL}`);
+    parts.push(`Title: ${context.currentPageInfo.title}`);
+    parts.push(`\nINTERACTIVE ELEMENTS (with positions and selectors):`);
+    parts.push(context.currentPageInfo.formattedElements);
+    parts.push(`\nARIA TREE (hierarchical structure):`);
+    parts.push(JSON.stringify(context.currentPageInfo.ariaSnapshot, null, 2).substring(0, 5000));
+    if (JSON.stringify(context.currentPageInfo.ariaSnapshot).length > 5000) {
+      parts.push('... (truncated)');
+    }
+    parts.push('')
+    
+    // Recent steps
+    if (context.recentSteps.length > 0) {
+      parts.push(`\nRECENT STEPS (last ${context.recentSteps.length}):`);
+      for (const step of context.recentSteps) {
+        const status = step.result === 'success' ? '✓' : '✗';
+        parts.push(`  ${status} ${step.stepNumber}.${step.iteration || ''} ${step.action}`);
+        parts.push(`     Code: ${step.code}`);
+        if (step.result === 'failure' && step.error) {
+          parts.push(`     ❌ ERROR: ${step.error}`);
+          parts.push(`     ^^ THIS SELECTOR FAILED - TRY DIFFERENT APPROACH ^^`);
+        } else {
+          parts.push(`     Result: ${step.observation}`);
+        }
+      }
+      parts.push('');
+      
+      // Detect repeated failures
+      const recentFailures = context.recentSteps.filter(s => s.result === 'failure');
+      if (recentFailures.length >= 2) {
+        const sameSelector = recentFailures.slice(-2).every((s, i, arr) => 
+          i === 0 || s.code === arr[i-1].code
+        );
+        if (sameSelector) {
+          parts.push(`⚠️ WARNING: You've tried the same selector multiple times and it failed!`);
+          parts.push(`   Last failed selector: ${recentFailures[recentFailures.length - 1].code}`);
+          parts.push(`   YOU MUST try a completely different selector this time!\n`);
+        }
+      }
+    }
+    
+    // Experiences
+    if (context.experiences.length > 0) {
+      parts.push(`\nLEARNINGS (${context.experiences.length}):`);
+      for (const exp of context.experiences) {
+        parts.push(`  - ${exp}`);
+      }
+      parts.push('');
+    }
+    
+    // Extracted data
+    if (Object.keys(context.extractedData).length > 0) {
+      parts.push(`\nEXTRACTED DATA:`);
+      for (const [key, value] of Object.entries(context.extractedData)) {
+        parts.push(`  ${key}: ${value}`);
+      }
+      parts.push('');
+    }
+    
+    // Previous reflection
+    if (context.previousIterationGuidance) {
+      parts.push(`\nYOUR PREVIOUS GUIDANCE TO YOURSELF:`);
+      parts.push(`Focus: ${context.previousIterationGuidance.guidanceForNext}`);
+      if (context.previousIterationGuidance.detectingLoop) {
+        parts.push(`⚠ YOU DETECTED A LOOP: ${context.previousIterationGuidance.loopReasoning}`);
+      }
+      parts.push('');
+    }
+    
+    // Tool results (if any)
+    if (context.toolResults) {
+      parts.push(`\n========== TOOL RESULTS ==========`);
+      for (const [toolName, result] of Object.entries(context.toolResults)) {
+        parts.push(`\n${toolName}: ${result.success ? '✓ SUCCESS' : '✗ FAILED'}`);
+        
+        if (result.learning) {
+          parts.push(`\n📊 ANALYSIS FROM ${toolName.toUpperCase()}:`);
+          parts.push(result.learning);
+          parts.push(`^^ USE THIS INFORMATION TO DECIDE YOUR NEXT COMMANDS ^^`);
+        }
+        
+        if (result.error) {
+          parts.push(`  Error: ${result.error}`);
+        }
+      }
+      parts.push(`\n===================================\n`);
+    }
+    
+    parts.push('=== WHAT DO YOU WANT TO DO? ===');
+    parts.push('Respond with JSON following the format specified in the system prompt.');
+    
+    return parts.join('\n');
+  }
+  
+  /**
+   * Parse agent decision from LLM response
+   */
+  private parseAgentDecision(response: string): AgentDecision {
+    try {
+      // Extract JSON from response
+      const jsonMatch = response.match(/\{[\s\S]*\}/);
+      if (!jsonMatch) {
+        throw new Error('No JSON found in response');
+      }
+      
+      const parsed = JSON.parse(jsonMatch[0]);
+      
+      // Validate required fields
+      if (!parsed.status || !parsed.reasoning) {
+        throw new Error('Missing required fields: status and reasoning');
+      }
+      
+      return parsed as AgentDecision;
+      
+    } catch (error: any) {
+      this.logger?.(`[Orchestrator] ✗ Failed to parse agent decision: ${error.message}`, 'error');
+      this.logger?.(`[Orchestrator] Raw response: ${response.substring(0, 200)}...`, 'log');
+      
+      // Return fallback
+      return {
+        status: 'stuck',
+        statusReasoning: 'Failed to parse agent response',
+        reasoning: `Parse error: ${error.message}`
+      };
+    }
+  }
+  
+  /**
+   * Execute tools
+   */
+  private async executeTools(
+    toolCalls: any[],
+    page: any,
+    memory: JourneyMemory,
+    stepNumber: number
+  ): Promise<Record<string, any>> {
+    this.logger?.(`[Orchestrator] 🔧 Executing ${toolCalls.length} tool(s)`);
+    
+    const results: Record<string, any> = {};
+    const toolContext: ToolExecutionContext = {
+      page,
+      memory,
+      stepNumber,
+      logger: this.logger
+    };
+    
+    for (const toolCall of toolCalls.slice(0, this.config.maxToolCallsPerIteration)) {
+      this.logger?.(`[Orchestrator]   ▶ ${toolCall.name}(${JSON.stringify(toolCall.params).substring(0, 50)}...)`);
+      
+      const result = await this.toolRegistry.execute(toolCall, toolContext);
+      results[toolCall.name] = result;
+      
+      if (result.success) {
+        this.logger?.(`[Orchestrator]   ✓ ${toolCall.name} succeeded`);
+      } else {
+        this.logger?.(`[Orchestrator]   ✗ ${toolCall.name} failed: ${result.error}`, 'error');
+      }
+    }
+    
+    return results;
+  }
+  
+  /**
+   * Execute commands sequentially with SHARED context (variables persist across commands)
+   */
+  private async executeCommandsSequentially(
+    commands: string[],
+    page: any,
+    memory: JourneyMemory,
+    stepNumber: number,
+    iteration: number,
+    jobId: string
+  ): Promise<{ executed: string[]; allSucceeded: boolean }> {
+    this.logger?.(`[Orchestrator] 📝 Executing ${commands.length} command(s) in shared context`);
+    
+    const executed: string[] = [];
+    const limitedCommands = commands.slice(0, this.config.maxCommandsPerIteration);
+    
+    // Build execution with shared context (all commands share scope - variables persist)
+    const commandsWithTracking = limitedCommands.map((cmd, i) => {
+      return `
+// Command ${i + 1}/${limitedCommands.length}
+try {
+  ${cmd}
+  __results.push({ index: ${i}, success: true });
+} catch (error) {
+  __results.push({ index: ${i}, success: false, error: error.message });
+  throw error; // Stop on first failure
+}`;
+    }).join('\n');
+    
+    const wrappedCode = `
+const __results = [];
+${commandsWithTracking}
+return __results;
+`;
+    
+    try {
+      // Execute in shared context - variables declared here persist for entire scenario
+      // Pass both page and expect to make Playwright assertions available
+      const func = new Function('page', 'expect', 'return (async () => { ' + wrappedCode + ' })()');
+      const results = await func(page, expect);
+      
+      // Record results for each command
+      for (let i = 0; i < limitedCommands.length; i++) {
+        const cmd = limitedCommands[i];
+        const result = results[i];
+        
+        if (result && result.success) {
+          this.logger?.(`[Orchestrator]   ✓ [${i + 1}/${limitedCommands.length}] Success`);
+          
+          // Record in history
+          memory.history.push({
+            stepNumber,
+            iteration,
+            action: `Command ${i + 1}/${limitedCommands.length}`,
+            code: cmd,
+            result: 'success',
+            observation: 'Executed successfully',
+            url: page.url(),
+            timestamp: Date.now()
+          });
+          
+          executed.push(cmd);
+        }
+      }
+      
+      // Cap history
+      if (memory.history.length > this.config.maxHistorySize) {
+        memory.history = memory.history.slice(-this.config.maxHistorySize);
+      }
+      
+      return { executed, allSucceeded: true };
+      
+    } catch (error: any) {
+      // One of the commands failed - find which one
+      const errorMessage = error.message || String(error);
+      
+      // Capture page state for debug logging
+      let pageStateDebug = '';
+      if (this.debugMode) {
+        try {
+          const pageInfo = await getEnhancedPageInfo(page);
+          pageStateDebug = `
+=== DEBUG: PAGE STATE AT FAILURE ===
+URL: ${page.url()}
+Title: ${pageInfo.title}
+
+INTERACTIVE ELEMENTS:
+${pageInfo.formattedElements}
+
+ARIA SNAPSHOT:
+${JSON.stringify(pageInfo.ariaSnapshot, null, 2)}
+
+====================================`;
+        } catch (debugError: any) {
+          pageStateDebug = `Failed to capture page state: ${debugError.message}`;
+        }
+      }
+      
+      // Record all that succeeded, then the failure
+      for (let i = 0; i < limitedCommands.length; i++) {
+        const cmd = limitedCommands[i];
+        
+        // This is a failed command (error happened here or earlier)
+        if (executed.length <= i) {
+          this.logger?.(`[Orchestrator]   ✗ [${i + 1}/${limitedCommands.length}] Failed: ${errorMessage}`, 'error');
+          
+          // Log detailed debug info
+          if (this.debugMode && pageStateDebug) {
+            this.logger?.(pageStateDebug, 'debug');
+          }
+          
+          memory.history.push({
+            stepNumber,
+            iteration,
+            action: `Command ${i + 1}/${limitedCommands.length} - FAILED`,
+            code: cmd,
+            result: 'failure',
+            observation: `Failed with error: ${errorMessage}. This selector likely doesn't exist or is incorrect.`,
+            error: errorMessage,
+            url: page.url(),
+            timestamp: Date.now()
+          });
+          
+          if (i < limitedCommands.length - 1) {
+            this.logger?.(`[Orchestrator]   ⚠ Skipping remaining ${limitedCommands.length - i - 1} command(s)`, 'warn');
+          }
+          
+          break;
+        }
+      }
+      
+      return { executed, allSucceeded: false };
+    }
+  }
+  
+  /**
+   * Execute a single command
+   */
+  private async executeCommand(cmd: string, page: any): Promise<void> {
+    // Wrap in async function and execute
+    const wrapped = `(async () => { ${cmd} })()`;
+    
+    try {
+      await eval(wrapped);
+    } catch (error: any) {
+      // If eval fails, try direct execution with page context
+      // Pass both page and expect to make Playwright assertions available
+      const func = new Function('page', 'expect', `return (async () => { ${cmd} })()`);
+      await func(page, expect);
+    }
+  }
+  
+  /**
+   * Log agent decision
+   */
+  private logAgentDecision(decision: AgentDecision, iteration: number): void {
+    this.logger?.(`[Orchestrator] 💭 REASONING: ${decision.reasoning}`);
+    
+    if (decision.selfReflection) {
+      this.logger?.(`[Orchestrator] 🧠 SELF-REFLECTION:`);
+      this.logger?.(`[Orchestrator]    Next: ${decision.selfReflection.guidanceForNext}`);
+      if (decision.selfReflection.detectingLoop) {
+        this.logger?.(`[Orchestrator]    🔄 LOOP DETECTED: ${decision.selfReflection.loopReasoning}`, 'warn');
+      }
+    }
+    
+    if (decision.toolCalls && decision.toolCalls.length > 0) {
+      this.logger?.(`[Orchestrator] 🔧 TOOLS: ${decision.toolCalls.map(t => t.name).join(', ')}`);
+      if (decision.toolReasoning) {
+        this.logger?.(`[Orchestrator] 📋 Why: ${decision.toolReasoning}`);
+      }
+    }
+    
+    if (decision.blockerDetected) {
+      this.logger?.(`[Orchestrator] 🚧 BLOCKER: ${decision.blockerDetected.description}`, 'warn');
+      this.logger?.(`[Orchestrator] 🧹 Clearing with ${decision.blockerDetected.clearingCommands.length} command(s)`);
+    }
+    
+    if (decision.stepReEvaluation?.detected) {
+      this.logger?.(`[Orchestrator] 🔍 STEP RE-EVALUATION: ${decision.stepReEvaluation.issue}`, 'warn');
+      this.logger?.(`[Orchestrator] 📝 Explanation: ${decision.stepReEvaluation.explanation}`);
+    }
+    
+    if (decision.commands && decision.commands.length > 0) {
+      this.logger?.(`[Orchestrator] 📝 COMMANDS (${decision.commands.length}):`);
+      decision.commands.slice(0, 3).forEach((cmd, i) => {
+        this.logger?.(`[Orchestrator]    ${i + 1}. ${cmd.substring(0, 80)}...`);
+      });
+      if (decision.commands.length > 3) {
+        this.logger?.(`[Orchestrator]    ... and ${decision.commands.length - 3} more`);
+      }
+      if (decision.commandReasoning) {
+        this.logger?.(`[Orchestrator] 💡 Why: ${decision.commandReasoning}`);
+      }
+    }
+    
+    // Experiences will be logged when added to memory, no need to log here
+  }
+  
+  /**
+   * Report step progress
+   */
+  private async reportStepProgress(
+    jobId: string,
+    stepNumber: number,
+    description: string,
+    decision: AgentDecision,
+    iteration: number
+  ): Promise<void> {
+    if (!this.progressReporter?.onStepProgress) return;
+    
+    await this.progressReporter.onStepProgress({
+      jobId,
+      stepNumber,
+      description,
+      status: decision.status === 'complete' ? StepExecutionStatus.SUCCESS :
+              decision.status === 'stuck' || decision.status === 'infeasible' ? StepExecutionStatus.FAILURE :
+              StepExecutionStatus.IN_PROGRESS,
+      code: decision.commands?.join('\n'),
+      // Include agent metadata for transparency
+      agentIteration: iteration,
+      agentReasoning: decision.reasoning,
+      agentSelfReflection: decision.selfReflection,
+      agentExperiences: decision.experiences,
+      agentToolsUsed: decision.toolCalls?.map(t => t.name),
+      agentStatus: decision.status
+    });
+  }
+}
+
+