testchimp-runner-core 0.0.33 → 0.0.34

package/src/orchestrator/types.ts

@@ -12,6 +12,42 @@ export interface SelfReflection {
   loopReasoning?: string;   // "Tried text-based selectors 3 times, switching to IDs"
 }
 
+/**
+ * Note to future self - iteration-specific memory for tactical continuity
+ * More flexible than structured SelfReflection - agent can write freely
+ */
+export interface NoteToFutureSelf {
+  fromIteration: number;
+  content: string;  // FREE-FORM - agent writes whatever tactical info it needs for next iteration
+  // Examples:
+  // - "Tried #menu-btn, failed. Will try SVG child next."
+  // - "Plan: Hover over menu to reveal dropdown, then click Profile option."
+  // - "Element loads async. Wait 2s after page load before clicking."
+}
+
+/**
+ * Coordinate-based action (fallback when selectors fail)
+ * Uses percentage-based positioning for resolution independence
+ */
+export interface CoordinateAction {
+  type: 'coordinate';
+  action: 'click' | 'doubleClick' | 'rightClick' | 'hover' | 'drag' | 'fill' | 'scroll';
+  
+  // Primary coordinates as percentages (0.0 to 100.0, 3 decimal precision for ~1px accuracy)
+  xPercent: number;  // 0 = left edge, 100 = right edge
+  yPercent: number;  // 0 = top edge, 100 = bottom edge
+  
+  // For drag actions
+  toXPercent?: number;
+  toYPercent?: number;
+  
+  // For fill actions (click then type)
+  value?: string;
+  
+  // For scroll actions
+  scrollAmount?: number;  // Positive = scroll down, negative = scroll up
+}
+
 /**
  * Journey memory - tracks the agent's journey through the scenario
  */
@@ -24,6 +60,9 @@ export interface JourneyMemory {
   
   // Data extracted and saved for later steps
   extractedData: Record<string, string>;
+  
+  // Latest note from agent (persists across steps for continuity)
+  latestNote?: NoteToFutureSelf;
 }
 
 /**
@@ -75,6 +114,12 @@ export interface AgentDecision {
   // Self-reflection for next iteration
   selfReflection?: SelfReflection;
   
+  // Note to future self (NEW - free-form iteration memory)
+  noteToFutureSelf?: string;  // Free-form tactical note for next iteration
+  
+  // Coordinate-based action (NEW - fallback when selectors fail)
+  coordinateAction?: CoordinateAction;
+  
   // Memory updates
   memoryUpdate?: {
     action: string;
@@ -130,6 +175,9 @@ export interface AgentContext {
   // Self-reflection from previous iteration
   previousIterationGuidance?: SelfReflection;
   
+  // Note from previous iteration (NEW - tactical continuity)
+  noteFromPreviousIteration?: NoteToFutureSelf;
+  
   // Tool results from this iteration (if any)
   toolResults?: Record<string, ToolResult>;
 }

package/src/prompts.ts

@@ -329,10 +329,10 @@ ${pageInfo.formattedElements}
        
        CRITICAL: Extract and use exact values mentioned in the goal.
        
-       Examples:
-       - Goal: "Login with credentials: Willy, Willy@1234"
-         ✅ Use: await page.fill('username', 'Willy'); await page.fill('password', 'Willy@1234');
-         ❌ NOT: await page.fill('username', process.env.USERNAME);
+      Examples:
+      - Goal: "Login with credentials: alice, pass123"
+        ✅ Use: await page.fill('username', 'alice'); await page.fill('password', 'pass123');
+        ❌ NOT: await page.fill('username', process.env.USERNAME);
        
        - Goal: "Enter name: John Doe"
          ✅ Use: await page.fill('[name="name"]', 'John Doe');
@@ -362,36 +362,42 @@ ${pageInfo.formattedElements}
        
        Playwright throws "strict mode violation" when a selector matches MULTIPLE elements.
        
-       **PROACTIVE DETECTION** - Check BEFORE generating command:
-       - Review the "CURRENT PAGE STATE" section below (accessibility tree / aria snapshot)
-       - Look for duplicate elements with same role/text (e.g., multiple links with "Employee Information")
-       - If duplicates exist, generate a MORE SPECIFIC selector from the start
-       - Don't wait for strict mode error - prevent it by analyzing the DOM structure
-       
-       🚨 ERROR PATTERNS:
-       - "strict mode violation" → Your selector matched multiple elements
-       - "Multiple elements found" → Same issue
-       - Command chain with multiple strategies → Sign of selector problems
-       
-       ✅ SOLUTIONS (in order of preference):
-       
-       **Option 1: Be More Specific** (BEST):
-       - BAD: page.locator('a', { hasText: 'Employee Information' }).click() → Matches multiple links
-       - GOOD: page.locator('nav a', { hasText: 'Employee Information' }).click() → Scoped to nav
-       - GOOD: page.getByRole('navigation').getByRole('link', { name: 'Employee Information' }).click() → Role-based scoping
-       - GOOD: page.locator('a[href*="/employee"]', { hasText: 'Employee Information' }).click() → Combined attributes
-       
-       **Option 2: Use Position-Based Selection**:
-       - If multiple matches are expected: page.locator('a', { hasText: 'Employee Information' }).first().click()
-       - Or use: .nth(0) for first, .last() for last
-       
-       **Option 3: Filter by Visibility/State**:
-       - page.locator('button', { hasText: 'Submit' }).filter({ hasNotText: 'Draft' }).click()
-       
-       🚫 **ANTI-PATTERNS (DON'T DO THIS)**:
-       - BAD: Chaining multiple selector strategies in one command with semicolons
-       - BAD: Using page.evaluate() to find/click elements (defeats Playwright's auto-waiting)
-       - GOOD: ONE clear, specific selector like page.locator('nav a', { hasText: 'Employee Information' }).click()
+      **PROACTIVE DETECTION** - Check BEFORE generating command:
+      - Review the "CURRENT PAGE STATE" section below (accessibility tree / aria snapshot)
+      - Look for duplicate elements with same role/text (e.g., multiple links with "Settings")
+      - If duplicates exist, generate a MORE SPECIFIC selector from the start
+      - Don't wait for strict mode error - prevent it by analyzing the DOM structure
+      
+      🚨 ERROR PATTERNS:
+      - "strict mode violation" → Your selector matched multiple elements
+      - "Multiple elements found" → Same issue
+      - "locator resolved to 2 elements" → Often one is a tooltip/popover with duplicate text
+      - Command chain with multiple strategies → Sign of selector problems
+      
+      ✅ SOLUTIONS (in order of preference):
+      
+      **Option 1: Use Role-Based Selectors** (BEST - avoids tooltips):
+      - BAD: page.getByText('Settings').click() → Matches button AND its tooltip
+      - GOOD: page.getByRole('button', { name: 'Settings' }).click() → Only matches button role
+      - GOOD: page.locator('button').getByText('Settings').click() → Scoped to button tag
+      - GOOD: page.locator('[role="button"]').getByText('Settings').click() → Scoped to role
+      
+      **Option 2: Scope to Container**:
+      - BAD: page.locator('a', { hasText: 'Settings' }).click() → Matches multiple links
+      - GOOD: page.locator('nav a', { hasText: 'Settings' }).click() → Scoped to nav
+      - GOOD: page.locator('a[href*="/settings"]').click() → Use unique attribute
+      
+      **Option 2: Use Position-Based Selection**:
+      - If multiple matches are expected: page.locator('a', { hasText: 'Settings' }).first().click()
+      - Or use: .nth(0) for first, .last() for last
+      
+      **Option 3: Filter by Visibility/State**:
+      - page.locator('button', { hasText: 'Submit' }).filter({ hasNotText: 'Draft' }).click()
+      
+      🚫 **ANTI-PATTERNS (DON'T DO THIS)**:
+      - BAD: Chaining multiple selector strategies in one command with semicolons
+      - BAD: Using page.evaluate() to find/click elements (defeats Playwright's auto-waiting)
+      - GOOD: ONE clear, specific selector like page.locator('nav a', { hasText: 'Settings' }).click()
        
        **When You See Strict Mode Errors:**
        1. Analyze - Why did my selector match multiple elements?

package/src/scenario-worker-class.ts

@@ -21,7 +21,8 @@ import {
   RecallHistoryTool,
   InspectPageTool,
   CheckPageReadyTool,
-  ExtractDataTool
+  ExtractDataTool,
+  VerifyActionResultTool
 } from './orchestrator';
 
 // Define a simple logging interface for compatibility
@@ -104,12 +105,16 @@ export class ScenarioWorker extends EventEmitter {
     const takeScreenshotTool = new TakeScreenshotTool();
     takeScreenshotTool.setLLMFacade(this.llmFacade);  // Inject LLM for vision analysis
     
-    // Register 5 information-gathering tools (state changes via Playwright commands)
+    const verifyActionTool = new VerifyActionResultTool();
+    verifyActionTool.setLLMFacade(this.llmFacade);  // Inject LLM for vision comparison
+    
+    // Register 6 information-gathering tools (state changes via Playwright commands)
     this.toolRegistry.register(takeScreenshotTool);
     this.toolRegistry.register(new RecallHistoryTool());
     this.toolRegistry.register(new InspectPageTool());
     this.toolRegistry.register(new CheckPageReadyTool());
     this.toolRegistry.register(new ExtractDataTool());
+    this.toolRegistry.register(verifyActionTool);
     
     // Create orchestrator agent
     this.orchestratorAgent = new OrchestratorAgent(

package/src/utils/coordinate-converter.ts

@@ -0,0 +1,162 @@
+/**
+ * Coordinate Converter Utility
+ * Converts percentage-based coordinates to pixel coordinates and generates Playwright commands
+ */
+
+import { CoordinateAction } from '../orchestrator/types';
+
+export class CoordinateConverter {
+  
+  /**
+   * Convert percentage coordinates to actual pixel coordinates
+   */
+  static percentToPixels(
+    xPercent: number,
+    yPercent: number,
+    viewportWidth: number,
+    viewportHeight: number
+  ): { x: number; y: number } {
+    return {
+      x: Math.round((xPercent / 100) * viewportWidth),
+      y: Math.round((yPercent / 100) * viewportHeight)
+    };
+  }
+  
+  /**
+   * Get viewport dimensions from page
+   */
+  static async getViewportSize(page: any): Promise<{ width: number; height: number }> {
+    return await page.evaluate((): { width: number; height: number } => {
+      const win = (globalThis as any).window;
+      return {
+        width: win.innerWidth as number,
+        height: win.innerHeight as number
+      };
+    });
+  }
+  
+  /**
+   * Convert coordinate action with percentages to Playwright commands
+   * Returns array of command strings
+   */
+  static async generateCommands(
+    action: CoordinateAction,
+    page: any
+  ): Promise<string[]> {
+    const viewport = await this.getViewportSize(page);
+    const { x, y } = this.percentToPixels(action.xPercent, action.yPercent, viewport.width, viewport.height);
+    
+    const commands: string[] = [];
+    
+    switch (action.action) {
+      case 'click':
+        commands.push(`await page.mouse.click(${x}, ${y});`);
+        break;
+        
+      case 'doubleClick':
+        commands.push(`await page.mouse.dblclick(${x}, ${y});`);
+        break;
+        
+      case 'rightClick':
+        commands.push(`await page.mouse.click(${x}, ${y}, { button: 'right' });`);
+        break;
+        
+      case 'hover':
+        commands.push(`await page.mouse.move(${x}, ${y});`);
+        break;
+        
+      case 'drag':
+        if (action.toXPercent === undefined || action.toYPercent === undefined) {
+          throw new Error('Drag action requires toXPercent and toYPercent');
+        }
+        const to = this.percentToPixels(action.toXPercent, action.toYPercent, viewport.width, viewport.height);
+        commands.push(`await page.mouse.move(${x}, ${y});`);
+        commands.push(`await page.mouse.down();`);
+        commands.push(`await page.mouse.move(${to.x}, ${to.y});`);
+        commands.push(`await page.mouse.up();`);
+        break;
+        
+      case 'fill':
+        if (!action.value) {
+          throw new Error('Fill action requires value');
+        }
+        // Click to focus, wait briefly, then type
+        commands.push(`await page.mouse.click(${x}, ${y});`);
+        commands.push(`await page.waitForTimeout(100);`);
+        commands.push(`await page.keyboard.type(${JSON.stringify(action.value)});`);
+        break;
+        
+      case 'scroll':
+        const scrollAmount = action.scrollAmount || 100;
+        // Move to position, then scroll
+        commands.push(`await page.mouse.move(${x}, ${y});`);
+        commands.push(`await page.mouse.wheel(0, ${scrollAmount});`);
+        break;
+        
+      default:
+        throw new Error(`Unknown coordinate action: ${action.action}`);
+    }
+    
+    return commands;
+  }
+  
+  /**
+   * Execute coordinate action directly on page
+   * Used during agent execution (converts and runs immediately)
+   */
+  static async executeAction(
+    action: CoordinateAction,
+    page: any
+  ): Promise<void> {
+    const viewport = await this.getViewportSize(page);
+    const { x, y } = this.percentToPixels(action.xPercent, action.yPercent, viewport.width, viewport.height);
+    
+    switch (action.action) {
+      case 'click':
+        await page.mouse.click(x, y);
+        break;
+        
+      case 'doubleClick':
+        await page.mouse.dblclick(x, y);
+        break;
+        
+      case 'rightClick':
+        await page.mouse.click(x, y, { button: 'right' });
+        break;
+        
+      case 'hover':
+        await page.mouse.move(x, y);
+        break;
+        
+      case 'drag':
+        if (action.toXPercent === undefined || action.toYPercent === undefined) {
+          throw new Error('Drag requires toXPercent and toYPercent');
+        }
+        const to = this.percentToPixels(action.toXPercent, action.toYPercent, viewport.width, viewport.height);
+        await page.mouse.move(x, y);
+        await page.mouse.down();
+        await page.mouse.move(to.x, to.y);
+        await page.mouse.up();
+        break;
+        
+      case 'fill':
+        if (!action.value) {
+          throw new Error('Fill requires value');
+        }
+        await page.mouse.click(x, y);
+        await page.waitForTimeout(100);
+        await page.keyboard.type(action.value);
+        break;
+        
+      case 'scroll':
+        const scrollAmount = action.scrollAmount || 100;
+        await page.mouse.move(x, y);
+        await page.mouse.wheel(0, scrollAmount);
+        break;
+        
+      default:
+        throw new Error(`Unknown coordinate action: ${action.action}`);
+    }
+  }
+}
+