testchimp-runner-core 0.0.32 → 0.0.34

package/src/orchestrator/types.ts

@@ -12,6 +12,42 @@ export interface SelfReflection {
   loopReasoning?: string;   // "Tried text-based selectors 3 times, switching to IDs"
 }
 
+/**
+ * Note to future self - iteration-specific memory for tactical continuity
+ * More flexible than structured SelfReflection - agent can write freely
+ */
+export interface NoteToFutureSelf {
+  fromIteration: number;
+  content: string;  // FREE-FORM - agent writes whatever tactical info it needs for next iteration
+  // Examples:
+  // - "Tried #menu-btn, failed. Will try SVG child next."
+  // - "Plan: Hover over menu to reveal dropdown, then click Profile option."
+  // - "Element loads async. Wait 2s after page load before clicking."
+}
+
+/**
+ * Coordinate-based action (fallback when selectors fail)
+ * Uses percentage-based positioning for resolution independence
+ */
+export interface CoordinateAction {
+  type: 'coordinate';
+  action: 'click' | 'doubleClick' | 'rightClick' | 'hover' | 'drag' | 'fill' | 'scroll';
+  
+  // Primary coordinates as percentages (0.0 to 100.0, 3 decimal precision for ~1px accuracy)
+  xPercent: number;  // 0 = left edge, 100 = right edge
+  yPercent: number;  // 0 = top edge, 100 = bottom edge
+  
+  // For drag actions
+  toXPercent?: number;
+  toYPercent?: number;
+  
+  // For fill actions (click then type)
+  value?: string;
+  
+  // For scroll actions
+  scrollAmount?: number;  // Positive = scroll down, negative = scroll up
+}
+
 /**
  * Journey memory - tracks the agent's journey through the scenario
  */
@@ -24,6 +60,9 @@ export interface JourneyMemory {
   
   // Data extracted and saved for later steps
   extractedData: Record<string, string>;
+  
+  // Latest note from agent (persists across steps for continuity)
+  latestNote?: NoteToFutureSelf;
 }
 
 /**
@@ -75,6 +114,12 @@ export interface AgentDecision {
   // Self-reflection for next iteration
   selfReflection?: SelfReflection;
   
+  // Note to future self (NEW - free-form iteration memory)
+  noteToFutureSelf?: string;  // Free-form tactical note for next iteration
+  
+  // Coordinate-based action (NEW - fallback when selectors fail)
+  coordinateAction?: CoordinateAction;
+  
   // Memory updates
   memoryUpdate?: {
     action: string;
@@ -130,6 +175,9 @@ export interface AgentContext {
   // Self-reflection from previous iteration
   previousIterationGuidance?: SelfReflection;
   
+  // Note from previous iteration (NEW - tactical continuity)
+  noteFromPreviousIteration?: NoteToFutureSelf;
+  
   // Tool results from this iteration (if any)
   toolResults?: Record<string, ToolResult>;
 }

package/src/prompts.ts

@@ -329,10 +329,10 @@ ${pageInfo.formattedElements}
        
        CRITICAL: Extract and use exact values mentioned in the goal.
        
-       Examples:
-       - Goal: "Login with credentials: Willy, Willy@1234"
-         ✅ Use: await page.fill('username', 'Willy'); await page.fill('password', 'Willy@1234');
-         ❌ NOT: await page.fill('username', process.env.USERNAME);
+      Examples:
+      - Goal: "Login with credentials: alice, pass123"
+        ✅ Use: await page.fill('username', 'alice'); await page.fill('password', 'pass123');
+        ❌ NOT: await page.fill('username', process.env.USERNAME);
        
        - Goal: "Enter name: John Doe"
          ✅ Use: await page.fill('[name="name"]', 'John Doe');
@@ -362,36 +362,42 @@ ${pageInfo.formattedElements}
        
        Playwright throws "strict mode violation" when a selector matches MULTIPLE elements.
        
-       **PROACTIVE DETECTION** - Check BEFORE generating command:
-       - Review the "CURRENT PAGE STATE" section below (accessibility tree / aria snapshot)
-       - Look for duplicate elements with same role/text (e.g., multiple links with "Employee Information")
-       - If duplicates exist, generate a MORE SPECIFIC selector from the start
-       - Don't wait for strict mode error - prevent it by analyzing the DOM structure
-       
-       🚨 ERROR PATTERNS:
-       - "strict mode violation" → Your selector matched multiple elements
-       - "Multiple elements found" → Same issue
-       - Command chain with multiple strategies → Sign of selector problems
-       
-       ✅ SOLUTIONS (in order of preference):
-       
-       **Option 1: Be More Specific** (BEST):
-       - BAD: page.locator('a', { hasText: 'Employee Information' }).click() → Matches multiple links
-       - GOOD: page.locator('nav a', { hasText: 'Employee Information' }).click() → Scoped to nav
-       - GOOD: page.getByRole('navigation').getByRole('link', { name: 'Employee Information' }).click() → Role-based scoping
-       - GOOD: page.locator('a[href*="/employee"]', { hasText: 'Employee Information' }).click() → Combined attributes
-       
-       **Option 2: Use Position-Based Selection**:
-       - If multiple matches are expected: page.locator('a', { hasText: 'Employee Information' }).first().click()
-       - Or use: .nth(0) for first, .last() for last
-       
-       **Option 3: Filter by Visibility/State**:
-       - page.locator('button', { hasText: 'Submit' }).filter({ hasNotText: 'Draft' }).click()
-       
-       🚫 **ANTI-PATTERNS (DON'T DO THIS)**:
-       - BAD: Chaining multiple selector strategies in one command with semicolons
-       - BAD: Using page.evaluate() to find/click elements (defeats Playwright's auto-waiting)
-       - GOOD: ONE clear, specific selector like page.locator('nav a', { hasText: 'Employee Information' }).click()
+      **PROACTIVE DETECTION** - Check BEFORE generating command:
+      - Review the "CURRENT PAGE STATE" section below (accessibility tree / aria snapshot)
+      - Look for duplicate elements with same role/text (e.g., multiple links with "Settings")
+      - If duplicates exist, generate a MORE SPECIFIC selector from the start
+      - Don't wait for strict mode error - prevent it by analyzing the DOM structure
+      
+      🚨 ERROR PATTERNS:
+      - "strict mode violation" → Your selector matched multiple elements
+      - "Multiple elements found" → Same issue
+      - "locator resolved to 2 elements" → Often one is a tooltip/popover with duplicate text
+      - Command chain with multiple strategies → Sign of selector problems
+      
+      ✅ SOLUTIONS (in order of preference):
+      
+      **Option 1: Use Role-Based Selectors** (BEST - avoids tooltips):
+      - BAD: page.getByText('Settings').click() → Matches button AND its tooltip
+      - GOOD: page.getByRole('button', { name: 'Settings' }).click() → Only matches button role
+      - GOOD: page.locator('button').getByText('Settings').click() → Scoped to button tag
+      - GOOD: page.locator('[role="button"]').getByText('Settings').click() → Scoped to role
+      
+      **Option 2: Scope to Container**:
+      - BAD: page.locator('a', { hasText: 'Settings' }).click() → Matches multiple links
+      - GOOD: page.locator('nav a', { hasText: 'Settings' }).click() → Scoped to nav
+      - GOOD: page.locator('a[href*="/settings"]').click() → Use unique attribute
+      
+      **Option 2: Use Position-Based Selection**:
+      - If multiple matches are expected: page.locator('a', { hasText: 'Settings' }).first().click()
+      - Or use: .nth(0) for first, .last() for last
+      
+      **Option 3: Filter by Visibility/State**:
+      - page.locator('button', { hasText: 'Submit' }).filter({ hasNotText: 'Draft' }).click()
+      
+      🚫 **ANTI-PATTERNS (DON'T DO THIS)**:
+      - BAD: Chaining multiple selector strategies in one command with semicolons
+      - BAD: Using page.evaluate() to find/click elements (defeats Playwright's auto-waiting)
+      - GOOD: ONE clear, specific selector like page.locator('nav a', { hasText: 'Settings' }).click()
        
        **When You See Strict Mode Errors:**
        1. Analyze - Why did my selector match multiple elements?
@@ -640,7 +646,7 @@ Return JSON: {"steps": [{"description": "...", "code": "..."}, ...]}`
 
   // Repair suggestion
   REPAIR_SUGGESTION: {
-    SYSTEM: 'You are an expert test automation engineer specializing in fixing failing Playwright tests. Analyze the current DOM state, error message, and step description to suggest the best repair action. Consider the failure history to avoid repeating the same mistakes.',
+    SYSTEM: 'You are an expert test automation engineer specializing in fixing failing Playwright tests. Analyze the current DOM state, error message, and step description to suggest the best repair action. Consider the failure history to avoid repeating the same mistakes. CRITICAL: Only use valid Playwright API methods.',
     
     USER: (stepDescription: string, stepCode: string, errorMessage: string, pageInfo: any, failureHistory: string, recentRepairs: string) => `Analyze this failing Playwright test step and suggest a repair action.
 
@@ -659,11 +665,55 @@ ${pageInfo.formattedElements}
 
     ${recentRepairs}
 
+    VALID PLAYWRIGHT API METHODS (DO NOT use methods not listed here):
+    
+    Locator Methods:
+    - .click({ force: true })  // Force click even if element is covered
+    - .click({ timeout: 10000 })  // Increase timeout
+    - .fill(value)
+    - .type(value, { delay: 100 })  // Type with delay
+    - .press('Enter')
+    - .selectOption(value)
+    - .check() / .uncheck()
+    - .scrollIntoViewIfNeeded()  // NOT scrollIntoView() - that doesn't exist!
+    - .waitFor({ state: 'visible' })
+    - .waitFor({ state: 'attached' })
+    - .isVisible()
+    - .isEnabled()
+    - .count()
+    - .first() / .last() / .nth(index)
+    - .filter({ hasText: 'text' })
+    
+    Page Methods:
+    - page.goto(url, { waitUntil: 'networkidle' })
+    - page.waitForLoadState('networkidle')
+    - page.waitForTimeout(ms)
+    - page.evaluate(() => window.scrollBy(0, 500))  // Scroll page
+    - page.getByRole(role, { name: 'text' })
+    - page.getByText('text')
+    - page.getByLabel('text')
+    - page.getByPlaceholder('text')
+    - page.getByTestId('id')
+    - page.locator('selector')
+    
+    COMMON FIXES FOR "element outside viewport":
+    - Use { force: true } option: await locator.click({ force: true });
+    - Scroll page first: await page.evaluate(() => window.scrollBy(0, 300));
+    - Wait longer: await locator.click({ timeout: 10000 });
+    - Click parent/container instead if element has overlays
+    
+    INVALID METHODS TO AVOID:
+    - ❌ .scrollIntoView() - DOES NOT EXIST (use .scrollIntoViewIfNeeded() instead)
+    - ❌ .scrollIntoView({ behavior: 'smooth' }) - WRONG API
+    - ❌ Any jQuery methods - This is Playwright, not jQuery
+    
     Choose the best repair action:
     1. MODIFY - Fix the current step with better selectors, waits, or logic
-    2. INSERT - Add a new step before the current one (e.g., wait for element, scroll into view)
+    2. INSERT - Add a new step before the current one (e.g., wait for element, handle popups)
     3. REMOVE - Skip this step entirely if it's not essential
 
+    IMPORTANT: Your repair code MUST use ONLY valid Playwright API methods listed above!
+
     Respond with JSON:
     {
       "shouldContinue": true/false,
@@ -672,7 +722,7 @@ ${pageInfo.formattedElements}
         "operation": "MODIFY|INSERT|REMOVE",
         "newStep": {
           "description": "step description",
-          "code": "await page.getByRole('button', { name: 'Submit' }).click();"
+          "code": "await page.getByRole('button', { name: 'Submit' }).click({ force: true });"
         }
       }
     }`

package/src/scenario-worker-class.ts

@@ -21,7 +21,8 @@ import {
   RecallHistoryTool,
   InspectPageTool,
   CheckPageReadyTool,
-  ExtractDataTool
+  ExtractDataTool,
+  VerifyActionResultTool
 } from './orchestrator';
 
 // Define a simple logging interface for compatibility
@@ -104,12 +105,16 @@ export class ScenarioWorker extends EventEmitter {
     const takeScreenshotTool = new TakeScreenshotTool();
     takeScreenshotTool.setLLMFacade(this.llmFacade);  // Inject LLM for vision analysis
     
-    // Register 5 information-gathering tools (state changes via Playwright commands)
+    const verifyActionTool = new VerifyActionResultTool();
+    verifyActionTool.setLLMFacade(this.llmFacade);  // Inject LLM for vision comparison
+    
+    // Register 6 information-gathering tools (state changes via Playwright commands)
     this.toolRegistry.register(takeScreenshotTool);
     this.toolRegistry.register(new RecallHistoryTool());
     this.toolRegistry.register(new InspectPageTool());
     this.toolRegistry.register(new CheckPageReadyTool());
     this.toolRegistry.register(new ExtractDataTool());
+    this.toolRegistry.register(verifyActionTool);
     
     // Create orchestrator agent
     this.orchestratorAgent = new OrchestratorAgent(

package/src/utils/coordinate-converter.ts

@@ -0,0 +1,162 @@
+/**
+ * Coordinate Converter Utility
+ * Converts percentage-based coordinates to pixel coordinates and generates Playwright commands
+ */
+
+import { CoordinateAction } from '../orchestrator/types';
+
+export class CoordinateConverter {
+  
+  /**
+   * Convert percentage coordinates to actual pixel coordinates
+   */
+  static percentToPixels(
+    xPercent: number,
+    yPercent: number,
+    viewportWidth: number,
+    viewportHeight: number
+  ): { x: number; y: number } {
+    return {
+      x: Math.round((xPercent / 100) * viewportWidth),
+      y: Math.round((yPercent / 100) * viewportHeight)
+    };
+  }
+  
+  /**
+   * Get viewport dimensions from page
+   */
+  static async getViewportSize(page: any): Promise<{ width: number; height: number }> {
+    return await page.evaluate((): { width: number; height: number } => {
+      const win = (globalThis as any).window;
+      return {
+        width: win.innerWidth as number,
+        height: win.innerHeight as number
+      };
+    });
+  }
+  
+  /**
+   * Convert coordinate action with percentages to Playwright commands
+   * Returns array of command strings
+   */
+  static async generateCommands(
+    action: CoordinateAction,
+    page: any
+  ): Promise<string[]> {
+    const viewport = await this.getViewportSize(page);
+    const { x, y } = this.percentToPixels(action.xPercent, action.yPercent, viewport.width, viewport.height);
+    
+    const commands: string[] = [];
+    
+    switch (action.action) {
+      case 'click':
+        commands.push(`await page.mouse.click(${x}, ${y});`);
+        break;
+        
+      case 'doubleClick':
+        commands.push(`await page.mouse.dblclick(${x}, ${y});`);
+        break;
+        
+      case 'rightClick':
+        commands.push(`await page.mouse.click(${x}, ${y}, { button: 'right' });`);
+        break;
+        
+      case 'hover':
+        commands.push(`await page.mouse.move(${x}, ${y});`);
+        break;
+        
+      case 'drag':
+        if (action.toXPercent === undefined || action.toYPercent === undefined) {
+          throw new Error('Drag action requires toXPercent and toYPercent');
+        }
+        const to = this.percentToPixels(action.toXPercent, action.toYPercent, viewport.width, viewport.height);
+        commands.push(`await page.mouse.move(${x}, ${y});`);
+        commands.push(`await page.mouse.down();`);
+        commands.push(`await page.mouse.move(${to.x}, ${to.y});`);
+        commands.push(`await page.mouse.up();`);
+        break;
+        
+      case 'fill':
+        if (!action.value) {
+          throw new Error('Fill action requires value');
+        }
+        // Click to focus, wait briefly, then type
+        commands.push(`await page.mouse.click(${x}, ${y});`);
+        commands.push(`await page.waitForTimeout(100);`);
+        commands.push(`await page.keyboard.type(${JSON.stringify(action.value)});`);
+        break;
+        
+      case 'scroll':
+        const scrollAmount = action.scrollAmount || 100;
+        // Move to position, then scroll
+        commands.push(`await page.mouse.move(${x}, ${y});`);
+        commands.push(`await page.mouse.wheel(0, ${scrollAmount});`);
+        break;
+        
+      default:
+        throw new Error(`Unknown coordinate action: ${action.action}`);
+    }
+    
+    return commands;
+  }
+  
+  /**
+   * Execute coordinate action directly on page
+   * Used during agent execution (converts and runs immediately)
+   */
+  static async executeAction(
+    action: CoordinateAction,
+    page: any
+  ): Promise<void> {
+    const viewport = await this.getViewportSize(page);
+    const { x, y } = this.percentToPixels(action.xPercent, action.yPercent, viewport.width, viewport.height);
+    
+    switch (action.action) {
+      case 'click':
+        await page.mouse.click(x, y);
+        break;
+        
+      case 'doubleClick':
+        await page.mouse.dblclick(x, y);
+        break;
+        
+      case 'rightClick':
+        await page.mouse.click(x, y, { button: 'right' });
+        break;
+        
+      case 'hover':
+        await page.mouse.move(x, y);
+        break;
+        
+      case 'drag':
+        if (action.toXPercent === undefined || action.toYPercent === undefined) {
+          throw new Error('Drag requires toXPercent and toYPercent');
+        }
+        const to = this.percentToPixels(action.toXPercent, action.toYPercent, viewport.width, viewport.height);
+        await page.mouse.move(x, y);
+        await page.mouse.down();
+        await page.mouse.move(to.x, to.y);
+        await page.mouse.up();
+        break;
+        
+      case 'fill':
+        if (!action.value) {
+          throw new Error('Fill requires value');
+        }
+        await page.mouse.click(x, y);
+        await page.waitForTimeout(100);
+        await page.keyboard.type(action.value);
+        break;
+        
+      case 'scroll':
+        const scrollAmount = action.scrollAmount || 100;
+        await page.mouse.move(x, y);
+        await page.mouse.wheel(0, scrollAmount);
+        break;
+        
+      default:
+        throw new Error(`Unknown coordinate action: ${action.action}`);
+    }
+  }
+}
+