testchimp-runner-core 0.0.33 → 0.0.35

package/src/orchestrator/types.ts

@@ -12,6 +12,42 @@ export interface SelfReflection {
   loopReasoning?: string;   // "Tried text-based selectors 3 times, switching to IDs"
 }
 
+/**
+ * Note to future self - iteration-specific memory for tactical continuity
+ * More flexible than structured SelfReflection - agent can write freely
+ */
+export interface NoteToFutureSelf {
+  fromIteration: number;
+  content: string;  // FREE-FORM - agent writes whatever tactical info it needs for next iteration
+  // Examples:
+  // - "Tried #menu-btn, failed. Will try SVG child next."
+  // - "Plan: Hover over menu to reveal dropdown, then click Profile option."
+  // - "Element loads async. Wait 2s after page load before clicking."
+}
+
+/**
+ * Coordinate-based action (fallback when selectors fail)
+ * Uses percentage-based positioning for resolution independence
+ */
+export interface CoordinateAction {
+  type: 'coordinate';
+  action: 'click' | 'doubleClick' | 'rightClick' | 'hover' | 'drag' | 'fill' | 'scroll';
+  
+  // Primary coordinates as percentages (0.0 to 100.0, 3 decimal precision for ~1px accuracy)
+  xPercent: number;  // 0 = left edge, 100 = right edge
+  yPercent: number;  // 0 = top edge, 100 = bottom edge
+  
+  // For drag actions
+  toXPercent?: number;
+  toYPercent?: number;
+  
+  // For fill actions (click then type)
+  value?: string;
+  
+  // For scroll actions
+  scrollAmount?: number;  // Positive = scroll down, negative = scroll up
+}
+
 /**
  * Journey memory - tracks the agent's journey through the scenario
  */
@@ -24,6 +60,9 @@ export interface JourneyMemory {
   
   // Data extracted and saved for later steps
   extractedData: Record<string, string>;
+  
+  // Latest note from agent (persists across steps for continuity)
+  latestNote?: NoteToFutureSelf;
 }
 
 /**
@@ -69,12 +108,18 @@ export interface AgentDecision {
   needsToolResults?: boolean;  // Wait for tool results before proceeding with commands
   
   // Command batch (executed sequentially)
-  commands?: string[];
+  commands?: string[];  // Plain Playwright commands
   commandReasoning?: string;
   
   // Self-reflection for next iteration
   selfReflection?: SelfReflection;
   
+  // Note to future self (NEW - free-form iteration memory)
+  noteToFutureSelf?: string;  // Free-form tactical note for next iteration
+  
+  // Coordinate-based action (NEW - fallback when selectors fail)
+  coordinateAction?: CoordinateAction;
+  
   // Memory updates
   memoryUpdate?: {
     action: string;
@@ -102,6 +147,12 @@ export interface AgentDecision {
     issue: 'prior_incomplete' | 'already_done' | 'wrong_order' | null;
     explanation: string;                   // Why agent thinks step order is off
   };
+  
+  // Meta-learning: Suggested prompt improvements based on journey learnings
+  debugInfo?: {
+    suggestedPromptUpdates?: string;      // Confident suggestions for improving system/user prompts
+    reasoning?: string;                    // Why these updates would help
+  };
 }
 
 /**
@@ -127,13 +178,34 @@ export interface AgentContext {
   experiences: string[];
   extractedData: Record<string, string>;
   
-  // Self-reflection from previous iteration
-  previousIterationGuidance?: SelfReflection;
+  // Note from previous iteration (tactical continuity)
+  noteFromPreviousIteration?: NoteToFutureSelf;
+  
+  // Test data / credentials for exploration
+  testDataPrompt?: string;
+  
+  // SoM (Set-of-Marks) screenshot with visual markers
+  somScreenshot?: string;  // Data URL of screenshot with SoM markers
+  somElementMap?: string;  // Text map of SoM IDs to element details for disambiguation
+  
+  // Repair mode context (undefined for script gen/exploration)
+  priorSteps?: string[];   // Steps completed before current (e.g., ["1. Navigate", "2. Login"])
+  nextSteps?: string[];    // Steps after current (e.g., ["5. Submit", "6. Verify"])
   
   // Tool results from this iteration (if any)
   toolResults?: Record<string, ToolResult>;
 }
 
+/**
+ * Exploration mode configuration
+ */
+export interface ExplorationMode {
+  enabled: boolean;                    // Whether exploration mode is active
+  explorationPrompt: string;           // Journey-specific focus: "Explore Dashboard and test all widgets"
+  testDataPrompt?: string;             // Test data, credentials context
+  maxExplorationSteps?: number;        // Budget limit (default: 50) - agent can stop earlier
+}
+
 /**
  * Configurable guardrails
  */
@@ -163,6 +235,15 @@ export interface AgentConfig {
   // Allowed actions
   allowedExplorationActions?: string[];   // Default: ['hover', 'click_info', 'click_menu', 'focus'] (Phase 2)
   allowedDomains?: string[];              // For navigate_to_url validation
+  
+  // Feature flags
+  enableCoordinateMode?: boolean;         // Default: false (experimental - disable until stable)
+  useSoM?: boolean;                       // Default: true (Set-of-Marks visual mode)
+  somUseSomIdBasedCommands?: boolean;     // Default: false (use semantic selectors first)
+  somRestrictCoordinates?: boolean;       // Default: false (if true, strongly discourage coord commands except as absolute last resort)
+  
+  // Exploration mode (NEW)
+  explorationMode?: ExplorationMode;
 }
 
 /**
@@ -195,6 +276,16 @@ export const DEFAULT_AGENT_CONFIG: Required<AgentConfig> = {
   commandTimeout: 30000,
   explorationTimeout: 2000,
   allowedExplorationActions: ['hover', 'click_info', 'click_menu', 'focus'],
-  allowedDomains: []
+  allowedDomains: [],
+  enableCoordinateMode: false,  // Disabled by default - experimental feature
+  useSoM: true,  // Enabled by default - use Set-of-Marks visual mode
+  somUseSomIdBasedCommands: false,  // Use semantic selectors first
+  somRestrictCoordinates: false,  // Allow coords as valid fallback (for exploration)
+  explorationMode: {
+    enabled: false,
+    explorationPrompt: '',
+    testDataPrompt: undefined,
+    maxExplorationSteps: 50
+  }
 };
 

package/src/prompts.ts

@@ -329,10 +329,10 @@ ${pageInfo.formattedElements}
        
        CRITICAL: Extract and use exact values mentioned in the goal.
        
-       Examples:
-       - Goal: "Login with credentials: Willy, Willy@1234"
-         ✅ Use: await page.fill('username', 'Willy'); await page.fill('password', 'Willy@1234');
-         ❌ NOT: await page.fill('username', process.env.USERNAME);
+      Examples:
+      - Goal: "Login with credentials: alice, pass123"
+        ✅ Use: await page.fill('username', 'alice'); await page.fill('password', 'pass123');
+        ❌ NOT: await page.fill('username', process.env.USERNAME);
        
        - Goal: "Enter name: John Doe"
          ✅ Use: await page.fill('[name="name"]', 'John Doe');
@@ -362,36 +362,42 @@ ${pageInfo.formattedElements}
        
        Playwright throws "strict mode violation" when a selector matches MULTIPLE elements.
        
-       **PROACTIVE DETECTION** - Check BEFORE generating command:
-       - Review the "CURRENT PAGE STATE" section below (accessibility tree / aria snapshot)
-       - Look for duplicate elements with same role/text (e.g., multiple links with "Employee Information")
-       - If duplicates exist, generate a MORE SPECIFIC selector from the start
-       - Don't wait for strict mode error - prevent it by analyzing the DOM structure
-       
-       🚨 ERROR PATTERNS:
-       - "strict mode violation" → Your selector matched multiple elements
-       - "Multiple elements found" → Same issue
-       - Command chain with multiple strategies → Sign of selector problems
-       
-       ✅ SOLUTIONS (in order of preference):
-       
-       **Option 1: Be More Specific** (BEST):
-       - BAD: page.locator('a', { hasText: 'Employee Information' }).click() → Matches multiple links
-       - GOOD: page.locator('nav a', { hasText: 'Employee Information' }).click() → Scoped to nav
-       - GOOD: page.getByRole('navigation').getByRole('link', { name: 'Employee Information' }).click() → Role-based scoping
-       - GOOD: page.locator('a[href*="/employee"]', { hasText: 'Employee Information' }).click() → Combined attributes
-       
-       **Option 2: Use Position-Based Selection**:
-       - If multiple matches are expected: page.locator('a', { hasText: 'Employee Information' }).first().click()
-       - Or use: .nth(0) for first, .last() for last
-       
-       **Option 3: Filter by Visibility/State**:
-       - page.locator('button', { hasText: 'Submit' }).filter({ hasNotText: 'Draft' }).click()
-       
-       🚫 **ANTI-PATTERNS (DON'T DO THIS)**:
-       - BAD: Chaining multiple selector strategies in one command with semicolons
-       - BAD: Using page.evaluate() to find/click elements (defeats Playwright's auto-waiting)
-       - GOOD: ONE clear, specific selector like page.locator('nav a', { hasText: 'Employee Information' }).click()
+      **PROACTIVE DETECTION** - Check BEFORE generating command:
+      - Review the "CURRENT PAGE STATE" section below (accessibility tree / aria snapshot)
+      - Look for duplicate elements with same role/text (e.g., multiple links with "Settings")
+      - If duplicates exist, generate a MORE SPECIFIC selector from the start
+      - Don't wait for strict mode error - prevent it by analyzing the DOM structure
+      
+      🚨 ERROR PATTERNS:
+      - "strict mode violation" → Your selector matched multiple elements
+      - "Multiple elements found" → Same issue
+      - "locator resolved to 2 elements" → Often one is a tooltip/popover with duplicate text
+      - Command chain with multiple strategies → Sign of selector problems
+      
+      ✅ SOLUTIONS (in order of preference):
+      
+      **Option 1: Use Role-Based Selectors** (BEST - avoids tooltips):
+      - BAD: page.getByText('Settings').click() → Matches button AND its tooltip
+      - GOOD: page.getByRole('button', { name: 'Settings' }).click() → Only matches button role
+      - GOOD: page.locator('button').getByText('Settings').click() → Scoped to button tag
+      - GOOD: page.locator('[role="button"]').getByText('Settings').click() → Scoped to role
+      
+      **Option 2: Scope to Container**:
+      - BAD: page.locator('a', { hasText: 'Settings' }).click() → Matches multiple links
+      - GOOD: page.locator('nav a', { hasText: 'Settings' }).click() → Scoped to nav
+      - GOOD: page.locator('a[href*="/settings"]').click() → Use unique attribute
+      
+      **Option 2: Use Position-Based Selection**:
+      - If multiple matches are expected: page.locator('a', { hasText: 'Settings' }).first().click()
+      - Or use: .nth(0) for first, .last() for last
+      
+      **Option 3: Filter by Visibility/State**:
+      - page.locator('button', { hasText: 'Submit' }).filter({ hasNotText: 'Draft' }).click()
+      
+      🚫 **ANTI-PATTERNS (DON'T DO THIS)**:
+      - BAD: Chaining multiple selector strategies in one command with semicolons
+      - BAD: Using page.evaluate() to find/click elements (defeats Playwright's auto-waiting)
+      - GOOD: ONE clear, specific selector like page.locator('nav a', { hasText: 'Settings' }).click()
        
        **When You See Strict Mode Errors:**
        1. Analyze - Why did my selector match multiple elements?

package/src/scenario-service.ts

@@ -218,6 +218,26 @@ export class ScenarioService extends EventEmitter {
     this.processNextJob();
   }
 
+  /**
+   * Execute exploration mode using orchestrator
+   * Requires orchestrator to be enabled via useOrchestrator option
+   */
+  async executeExploration(page: any, explorationConfig: any, jobId: string): Promise<any> {
+    if (!this.useOrchestrator) {
+      throw new Error('Exploration mode requires orchestrator to be enabled');
+    }
+    
+    // Get an available worker (or create one if needed)
+    let worker = this.workers.find(w => !this.busyWorkers.has(w));
+    if (!worker) {
+      await this.createWorker();
+      worker = this.workers[this.workers.length - 1];
+    }
+    
+    // Execute exploration via worker's orchestrator
+    return worker.executeExploration(page, explorationConfig, jobId);
+  }
+
   async shutdown(): Promise<void> {
     this.log('Shutting down scenario service...');
     

package/src/scenario-worker-class.ts

@@ -18,10 +18,13 @@ import {
   JourneyMemory,
   AgentConfig,
   TakeScreenshotTool,
+  ViewPreviousScreenshotTool,
+  RefreshSomMarkersTool,
   RecallHistoryTool,
   InspectPageTool,
   CheckPageReadyTool,
-  ExtractDataTool
+  ExtractDataTool,
+  VerifyActionResultTool
 } from './orchestrator';
 
 // Define a simple logging interface for compatibility
@@ -104,12 +107,23 @@ export class ScenarioWorker extends EventEmitter {
     const takeScreenshotTool = new TakeScreenshotTool();
     takeScreenshotTool.setLLMFacade(this.llmFacade);  // Inject LLM for vision analysis
     
-    // Register 5 information-gathering tools (state changes via Playwright commands)
+    const viewPreviousScreenshotTool = new ViewPreviousScreenshotTool();
+    viewPreviousScreenshotTool.setLLMFacade(this.llmFacade);  // Inject LLM for vision analysis
+    
+    const refreshSomMarkersTool = new RefreshSomMarkersTool();
+    
+    const verifyActionTool = new VerifyActionResultTool();
+    verifyActionTool.setLLMFacade(this.llmFacade);  // Inject LLM for vision comparison
+    
+    // Register 8 information-gathering tools (state changes via Playwright commands)
     this.toolRegistry.register(takeScreenshotTool);
+    this.toolRegistry.register(viewPreviousScreenshotTool);
+    this.toolRegistry.register(refreshSomMarkersTool);
     this.toolRegistry.register(new RecallHistoryTool());
     this.toolRegistry.register(new InspectPageTool());
     this.toolRegistry.register(new CheckPageReadyTool());
     this.toolRegistry.register(new ExtractDataTool());
+    this.toolRegistry.register(verifyActionTool);
     
     // Create orchestrator agent
     this.orchestratorAgent = new OrchestratorAgent(
@@ -312,12 +326,12 @@ export class ScenarioWorker extends EventEmitter {
         page = job.existingPage;
       } else {
         // Create new browser (default behavior for local clients)
-        // Default to headed mode (headless: false) for better debugging
+        // Let the playwrightConfig control headless mode (don't override with hardcoded value)
         // Create logger function from outputChannel for browser initialization
         const logger = this.outputChannel ? (message: string, level?: 'log' | 'error' | 'warn') => {
           this.outputChannel!.appendLine(`[Browser] ${message}`);
         } : undefined;
-        const browserInstance = await initializeBrowser(job.playwrightConfig, false, undefined, logger);
+        const browserInstance = await initializeBrowser(job.playwrightConfig, undefined, undefined, logger);
         browser = browserInstance.browser;
         context = browserInstance.context;
         page = browserInstance.page;
@@ -1077,6 +1091,18 @@ export class ScenarioWorker extends EventEmitter {
 
 
 
+  /**
+   * Execute exploration mode using orchestrator
+   */
+  async executeExploration(page: any, explorationConfig: any, jobId: string): Promise<any> {
+    if (!this.useOrchestrator || !this.orchestratorAgent) {
+      throw new Error('Orchestrator not available - exploration mode requires orchestrator');
+    }
+    
+    // Execute exploration via orchestrator
+    return this.orchestratorAgent.executeExploration(page, explorationConfig, jobId);
+  }
+
   async cleanup(): Promise<void> {
     this.initialized = false;
     this.sessionId = null;

package/src/utils/coordinate-converter.ts

@@ -0,0 +1,162 @@
+/**
+ * Coordinate Converter Utility
+ * Converts percentage-based coordinates to pixel coordinates and generates Playwright commands
+ */
+
+import { CoordinateAction } from '../orchestrator/types';
+
+export class CoordinateConverter {
+  
+  /**
+   * Convert percentage coordinates to actual pixel coordinates
+   */
+  static percentToPixels(
+    xPercent: number,
+    yPercent: number,
+    viewportWidth: number,
+    viewportHeight: number
+  ): { x: number; y: number } {
+    return {
+      x: Math.round((xPercent / 100) * viewportWidth),
+      y: Math.round((yPercent / 100) * viewportHeight)
+    };
+  }
+  
+  /**
+   * Get viewport dimensions from page
+   */
+  static async getViewportSize(page: any): Promise<{ width: number; height: number }> {
+    return await page.evaluate((): { width: number; height: number } => {
+      const win = (globalThis as any).window;
+      return {
+        width: win.innerWidth as number,
+        height: win.innerHeight as number
+      };
+    });
+  }
+  
+  /**
+   * Convert coordinate action with percentages to Playwright commands
+   * Returns array of command strings
+   */
+  static async generateCommands(
+    action: CoordinateAction,
+    page: any
+  ): Promise<string[]> {
+    const viewport = await this.getViewportSize(page);
+    const { x, y } = this.percentToPixels(action.xPercent, action.yPercent, viewport.width, viewport.height);
+    
+    const commands: string[] = [];
+    
+    switch (action.action) {
+      case 'click':
+        commands.push(`await page.mouse.click(${x}, ${y});`);
+        break;
+        
+      case 'doubleClick':
+        commands.push(`await page.mouse.dblclick(${x}, ${y});`);
+        break;
+        
+      case 'rightClick':
+        commands.push(`await page.mouse.click(${x}, ${y}, { button: 'right' });`);
+        break;
+        
+      case 'hover':
+        commands.push(`await page.mouse.move(${x}, ${y});`);
+        break;
+        
+      case 'drag':
+        if (action.toXPercent === undefined || action.toYPercent === undefined) {
+          throw new Error('Drag action requires toXPercent and toYPercent');
+        }
+        const to = this.percentToPixels(action.toXPercent, action.toYPercent, viewport.width, viewport.height);
+        commands.push(`await page.mouse.move(${x}, ${y});`);
+        commands.push(`await page.mouse.down();`);
+        commands.push(`await page.mouse.move(${to.x}, ${to.y});`);
+        commands.push(`await page.mouse.up();`);
+        break;
+        
+      case 'fill':
+        if (!action.value) {
+          throw new Error('Fill action requires value');
+        }
+        // Click to focus, wait briefly, then type
+        commands.push(`await page.mouse.click(${x}, ${y});`);
+        commands.push(`await page.waitForTimeout(100);`);
+        commands.push(`await page.keyboard.type(${JSON.stringify(action.value)});`);
+        break;
+        
+      case 'scroll':
+        const scrollAmount = action.scrollAmount || 100;
+        // Move to position, then scroll
+        commands.push(`await page.mouse.move(${x}, ${y});`);
+        commands.push(`await page.mouse.wheel(0, ${scrollAmount});`);
+        break;
+        
+      default:
+        throw new Error(`Unknown coordinate action: ${action.action}`);
+    }
+    
+    return commands;
+  }
+  
+  /**
+   * Execute coordinate action directly on page
+   * Used during agent execution (converts and runs immediately)
+   */
+  static async executeAction(
+    action: CoordinateAction,
+    page: any
+  ): Promise<void> {
+    const viewport = await this.getViewportSize(page);
+    const { x, y } = this.percentToPixels(action.xPercent, action.yPercent, viewport.width, viewport.height);
+    
+    switch (action.action) {
+      case 'click':
+        await page.mouse.click(x, y);
+        break;
+        
+      case 'doubleClick':
+        await page.mouse.dblclick(x, y);
+        break;
+        
+      case 'rightClick':
+        await page.mouse.click(x, y, { button: 'right' });
+        break;
+        
+      case 'hover':
+        await page.mouse.move(x, y);
+        break;
+        
+      case 'drag':
+        if (action.toXPercent === undefined || action.toYPercent === undefined) {
+          throw new Error('Drag requires toXPercent and toYPercent');
+        }
+        const to = this.percentToPixels(action.toXPercent, action.toYPercent, viewport.width, viewport.height);
+        await page.mouse.move(x, y);
+        await page.mouse.down();
+        await page.mouse.move(to.x, to.y);
+        await page.mouse.up();
+        break;
+        
+      case 'fill':
+        if (!action.value) {
+          throw new Error('Fill requires value');
+        }
+        await page.mouse.click(x, y);
+        await page.waitForTimeout(100);
+        await page.keyboard.type(action.value);
+        break;
+        
+      case 'scroll':
+        const scrollAmount = action.scrollAmount || 100;
+        await page.mouse.move(x, y);
+        await page.mouse.wheel(0, scrollAmount);
+        break;
+        
+      default:
+        throw new Error(`Unknown coordinate action: ${action.action}`);
+    }
+  }
+}
+

package/src/utils/page-info-retry.ts

@@ -0,0 +1,65 @@
+/**
+ * Page Info Retry Utility
+ * Handles adaptive page loading with exponential backoff
+ */
+
+import { getEnhancedPageInfo, PageInfo } from './page-info-utils';
+
+export class PageInfoRetry {
+  /**
+   * Get page info with retry logic - waits for interactive elements to appear
+   * Uses exponential backoff to handle slow-loading React/Vue/Angular apps
+   */
+  static async getWithRetry(page: any, maxAttempts: number = 6): Promise<PageInfo> {
+    // Wait for initial page load (generous timeout for slow apps)
+    try {
+      await page.waitForLoadState('domcontentloaded', { timeout: 20000 }).catch(() => {});
+    } catch (waitError) {
+      // Continue even if wait fails
+    }
+    
+    let attempt = 0;
+    let backoffMs = 1000; // Start with 1 second (adequate for most sites)
+    
+    while (attempt < maxAttempts) {
+      attempt++;
+      
+      // Try to extract page info
+      const pageInfo = await getEnhancedPageInfo(page);
+      
+      // If we got a reasonable number of elements, we're done
+      if (pageInfo.interactiveElements && pageInfo.interactiveElements.length >= 3) {
+        if (attempt > 1) {
+          console.log(`[PageInfoRetry] ✓ Page elements loaded after ${attempt} attempts`);
+        }
+        return pageInfo;
+      }
+      
+      // If this is the last attempt, return what we have
+      if (attempt >= maxAttempts) {
+        const totalWait = this.calculateTotalWaitTime(maxAttempts);
+        console.log(`[PageInfoRetry] ⚠️  Only found ${pageInfo.interactiveElements?.length || 0} elements after ${maxAttempts} attempts (total wait: ~${totalWait}ms)`);
+        return pageInfo;
+      }
+      
+      // Wait with exponential backoff before retrying
+      console.log(`[PageInfoRetry] Only ${pageInfo.interactiveElements?.length || 0} elements found (attempt ${attempt}/${maxAttempts}), waiting ${backoffMs}ms...`);
+      await page.waitForTimeout(backoffMs);
+      backoffMs = Math.min(backoffMs * 1.6, 15000); // Cap at 15 seconds per attempt
+    }
+    
+    // Fallback (shouldn't reach here, but for type safety)
+    return await getEnhancedPageInfo(page);
+  }
+  
+  private static calculateTotalWaitTime(maxAttempts: number): number {
+    let total = 0;
+    let backoffMs = 1000;
+    for (let i = 1; i < maxAttempts; i++) {
+      total += backoffMs;
+      backoffMs = Math.min(backoffMs * 1.6, 15000);
+    }
+    return Math.round(total);
+  }
+}
+