testchimp-runner-core 0.0.33 → 0.0.35

package/src/orchestrator/som-types.ts

@@ -0,0 +1,188 @@
+/**
+ * Set-of-Marks (SoM) Type Definitions
+ * Types for visual element identification and interaction
+ */
+
+export interface Coordinate {
+  x: number;  // Percentage of viewport width: 0-100 (use 3 decimal precision, e.g., 15.625)
+  y: number;  // Percentage of viewport height: 0-100 (use 3 decimal precision, e.g., 82.375)
+}
+
+export enum InteractionAction {
+  // Click actions
+  CLICK = 'click',
+  DOUBLE_CLICK = 'doubleClick',
+  RIGHT_CLICK = 'rightClick',
+  
+  // Mouse actions
+  HOVER = 'hover',
+  MOUSE_DOWN = 'mouseDown',
+  MOUSE_UP = 'mouseUp',
+  DRAG = 'drag',
+  
+  // Input actions
+  FILL = 'fill',
+  TYPE = 'type',
+  CLEAR = 'clear',
+  
+  // Keyboard actions
+  PRESS = 'press',
+  PRESS_SEQUENTIALLY = 'pressSequentially',
+  
+  // Select/Checkbox actions
+  SELECT = 'select',
+  CHECK = 'check',
+  UNCHECK = 'uncheck',
+  
+  // Focus/Scroll actions
+  FOCUS = 'focus',
+  BLUR = 'blur',
+  SCROLL = 'scroll',
+  SCROLL_INTO_VIEW = 'scrollIntoView',
+  
+  // Navigation actions
+  NAVIGATE = 'navigate',  // Go to URL (requires value field)
+  GO_BACK = 'goBack',
+  GO_FORWARD = 'goForward',
+  RELOAD = 'reload'
+}
+
+export interface SomCommand {
+  elementRef?: string;    // Integer as string: "1", "2", "42" (optional for coord-based commands)
+  action: InteractionAction;
+  
+  // Coordinate-based action (use when elementRef is empty/null)
+  coord?: Coordinate;     // Percentage-based (x: 0-100, y: 0-100 of viewport)
+  
+  // Action-specific parameters
+  value?: string;         // For fill/type/select/press actions
+  fromCoord?: Coordinate; // For drag (start) - percentage-based
+  toCoord?: Coordinate;   // For drag (end) - percentage-based
+  force?: boolean;        // Force action even if not actionable
+  scrollAmount?: number;  // Pixels to scroll
+  scrollDirection?: 'up' | 'down' | 'left' | 'right';
+  button?: 'left' | 'right' | 'middle';
+  clickCount?: number;
+  modifiers?: Array<'Alt' | 'Control' | 'Meta' | 'Shift'>;
+  delay?: number;         // Delay between keystrokes for TYPE (ms)
+  timeout?: number;       // Override default timeout
+}
+
+export enum CommandRunStatus {
+  SUCCESS = 'success',
+  FAILURE = 'failure'
+}
+
+export interface CommandAttempt {
+  command?: string;
+  status: CommandRunStatus;
+  error?: string;
+}
+
+export interface DomMutation {
+  type: 'added' | 'removed' | 'modified' | 'attribute_changed';
+  elementDescription: string;
+  timestamp: number;
+}
+
+export interface SemanticCommandResult {
+  failedAttempts: CommandAttempt[];
+  successAttempt?: CommandAttempt;
+  error?: string;
+  status: CommandRunStatus;
+  mutations?: DomMutation[];  // Only for hover/focus, filtered for relevance
+}
+
+export interface SomElement {
+  somId: string;  // Simple integer as string: "1", "2", "3"
+  tag: string;
+  role: string;
+  text: string;
+  ariaLabel: string;
+  placeholder: string;
+  name: string;
+  type: string;
+  id: string;
+  className: string;
+  bbox: { x: number; y: number; width: number; height: number };
+  parent?: {
+    tag: string;
+    role: string;
+    className: string;
+    text: string;
+  };
+}
+
+/**
+ * Typed selector (no string parsing needed)
+ * Supports chaining: parent.child for scoped selectors
+ */
+export interface TypedSelector {
+  type: 'id' | 'testId' | 'label' | 'role' | 'placeholder' | 'text' | 'title' | 'altText' | 'name' | 'locator';
+  value: string;
+  roleOptions?: { name?: string };  // For getByRole
+  parent?: TypedSelector;  // For chaining: page.locator(parent).locator(this)
+}
+
+/**
+ * Verification types for expect assertions
+ */
+export enum VerificationType {
+  // Text verifications
+  TEXT_CONTAINS = 'textContains',
+  TEXT_EQUALS = 'textEquals',
+  
+  // Input verifications
+  VALUE_EQUALS = 'valueEquals',
+  VALUE_EMPTY = 'valueEmpty',
+  
+  // Visibility verifications
+  IS_VISIBLE = 'isVisible',
+  IS_HIDDEN = 'isHidden',
+  
+  // State verifications
+  IS_ENABLED = 'isEnabled',
+  IS_DISABLED = 'isDisabled',
+  IS_CHECKED = 'isChecked',
+  IS_UNCHECKED = 'isUnchecked',
+  
+  // Count verifications (for lists, tables, etc.)
+  COUNT_EQUALS = 'countEquals',
+  COUNT_GREATER_THAN = 'countGreaterThan',
+  COUNT_LESS_THAN = 'countLessThan',
+  
+  // Attribute verifications
+  HAS_CLASS = 'hasClass',
+  HAS_ATTRIBUTE = 'hasAttribute'
+}
+
+/**
+ * SoM verification command for expect assertions
+ */
+export interface SomVerification {
+  verificationType: VerificationType;
+  elementRef?: string;       // SoM ID (e.g., "3") - optional for count verifications
+  expected?: string | number;  // Expected value/text/count
+  description?: string;        // Human-readable description
+  selector?: string;          // For count verifications on non-SoM elements (CSS selector)
+}
+
+/**
+ * Union type: commands array can contain both actions and verifications
+ */
+export type SomCommandOrVerification = SomCommand | SomVerification;
+
+/**
+ * Type guard to check if command is a verification
+ */
+export function isSomVerification(cmd: SomCommandOrVerification): cmd is SomVerification {
+  return 'verificationType' in cmd;
+}
+
+/**
+ * Type guard to check if command is an action
+ */
+export function isSomCommand(cmd: SomCommandOrVerification): cmd is SomCommand {
+  return 'action' in cmd;
+}
+

package/src/orchestrator/tool-registry.ts

@@ -40,6 +40,8 @@ export interface ToolExecutionContext {
   memory: any;  // JourneyMemory
   stepNumber: number;
   logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
+  previousSomScreenshot?: string;  // For view_previous_screenshot tool
+  somHandler?: any;  // PageSoMHandler for refresh_som_markers tool
 }
 
 /**

package/src/orchestrator/tools/index.ts

@@ -1,11 +1,15 @@
 /**
- * Tool exports - 5 information-gathering tools
+ * Tool exports - 8 information-gathering tools
  * Note: State changes (navigation, clicks, fills) are done via Playwright commands, not tools
+ * Ref-based commands (getByRef) are translated to Playwright at execution time
  */
 
 export { TakeScreenshotTool } from './take-screenshot';
+export { ViewPreviousScreenshotTool } from './view-previous-screenshot';
+export { RefreshSomMarkersTool } from './refresh-som-markers';
 export { RecallHistoryTool } from './recall-history';
 export { InspectPageTool } from './inspect-page';
 export { CheckPageReadyTool } from './check-page-ready';
 export { ExtractDataTool } from './extract-data';
+export { VerifyActionResultTool } from './verify-action-result';
 

package/src/orchestrator/tools/refresh-som-markers.ts

@@ -0,0 +1,69 @@
+import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
+import { ToolResult } from '../types';
+
+/**
+ * Tool to refresh SoM markers when they appear outdated or misaligned
+ */
+export class RefreshSomMarkersTool implements Tool {
+  name = 'refresh_som_markers';
+  description = 'Manually refresh the Set-of-Marks visual markers on the page. Use when: (1) Markers appear misaligned with actual UI elements, (2) Page content has changed but markers are stale (e.g., after dynamic content loads), (3) You suspect markers are from a previous page state. Returns updated screenshot with fresh markers.';
+  
+  parameters: ToolParameter[] = [
+    {
+      name: 'reason',
+      type: 'string',
+      description: 'Why you need to refresh markers. Examples: "dropdown expanded but markers still show closed state", "new content loaded but not marked", "markers seem to point to wrong elements"',
+      required: true
+    }
+  ];
+  
+  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
+    const { logger, page, somHandler } = context;
+    const reason = params.reason || 'Markers appear outdated';
+    
+    if (!somHandler) {
+      return {
+        success: false,
+        error: 'SoM mode not enabled - refresh markers tool unavailable.'
+      };
+    }
+    
+    if (!page) {
+      return {
+        success: false,
+        error: 'No page context available.'
+      };
+    }
+    
+    try {
+      logger?.(`[RefreshSomMarkers] Refreshing markers due to: ${reason}`, 'log');
+      
+      // Ensure somHandler has the latest page reference
+      somHandler.setPage(page);
+      
+      // Re-scan page and update markers
+      await somHandler.updateSom();
+      logger?.(`[RefreshSomMarkers] ✓ Markers updated`, 'log');
+      
+      // Capture fresh screenshot with new markers (viewport only - cheaper than full page)
+      const freshScreenshot = await somHandler.getScreenshot(true, false, 60);
+      logger?.(`[RefreshSomMarkers] ✓ Fresh screenshot captured (viewport)`, 'log');
+      
+      return {
+        success: true,
+        data: {
+          screenshot: freshScreenshot,
+          reason
+        },
+        learning: `SoM markers refreshed. New screenshot shows current page state with updated element markers. Reason: ${reason}`
+      };
+    } catch (error: any) {
+      logger?.(`[RefreshSomMarkers] ✗ Failed: ${error.message}`, 'error');
+      return {
+        success: false,
+        error: `Failed to refresh markers: ${error.message}`
+      };
+    }
+  }
+}
+

package/src/orchestrator/tools/verify-action-result.ts

@@ -0,0 +1,159 @@
+/**
+ * Verify Action Result Tool
+ * 
+ * Compares before/after screenshots to verify if an action achieved its goal
+ * USE SPARINGLY - expensive (sends 2 images to vision model)
+ */
+
+import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
+import { ToolResult } from '../types';
+import { LabeledImage } from '../../llm-provider';
+
+export class VerifyActionResultTool implements Tool {
+  name = 'verify_action_result';
+  
+  description = `Verify if your previous action achieved its goal by comparing before/after screenshots.
+
+⚠️ USE SPARINGLY - EXPENSIVE (2 images sent to vision model)
+
+When to use:
+- After coordinate click to verify expected UI change occurred
+- After clicking element when unsure if goal achieved (no clear feedback)
+- To verify navigation or modal appeared
+
+How it works:
+1. Takes screenshot BEFORE you run your next command
+2. You run command (click, fill, etc.)  
+3. Takes screenshot AFTER command executes
+4. Compares both and tells you if expected change happened
+
+Parameters:
+- expectedChange: What should have changed (e.g., "Dashboard page loaded", "Modal appeared", "Form submitted")
+
+Returns: { verified: boolean, reasoning: string, changes: string[] }
+
+Example:
+{
+  "toolCalls": [{"name": "verify_action_result", "params": {"expectedChange": "Dashboard page loaded with data grid"}}],
+  "toolReasoning": "Need to verify coordinate click navigated to correct page",
+  "needsToolResults": true  // IMPORTANT: Wait for verification before deciding status
+}`;
+
+  parameters: ToolParameter[] = [
+    {
+      name: 'expectedChange',
+      type: 'string',
+      description: 'What UI change you expect to see (e.g., "Modal opened", "Page navigated", "New form appeared")',
+      required: true
+    }
+  ];
+  
+  // LLM facade will be injected
+  private llmFacade?: any;
+  
+  setLLMFacade(llmFacade: any): void {
+    this.llmFacade = llmFacade;
+  }
+
+  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
+    const { page, logger } = context;
+    const expectedChange = params.expectedChange || 'Expected UI change';
+    
+    logger?.(`[VerifyActionResult] Capturing before/after screenshots...`);
+    
+    try {
+      // Capture BEFORE screenshot (viewport only, JPEG 60% quality for cost efficiency)
+      const beforeScreenshot = await page.screenshot({ 
+        encoding: 'base64', 
+        fullPage: false,
+        type: 'jpeg',
+        quality: 60
+      });
+      const beforeDataUrl = `data:image/jpeg;base64,${beforeScreenshot}`;
+      
+      logger?.(`[VerifyActionResult] ✓ BEFORE screenshot captured (JPEG 60%)`);
+      logger?.(`[VerifyActionResult] ⏳ Waiting 1.5s for UI to settle after your previous action...`);
+      
+      // Wait for UI to settle after previous action
+      await page.waitForTimeout(1500);
+      
+      // Capture AFTER screenshot
+      const afterScreenshot = await page.screenshot({ 
+        encoding: 'base64', 
+        fullPage: false,
+        type: 'jpeg',
+        quality: 60
+      });
+      const afterDataUrl = `data:image/jpeg;base64,${afterScreenshot}`;
+      
+      logger?.(`[VerifyActionResult] ✓ AFTER screenshot captured (JPEG 60%)`);
+      logger?.(`[VerifyActionResult] 🔍 Calling vision model to compare...`);
+      
+      if (!this.llmFacade) {
+        throw new Error('LLM facade not initialized');
+      }
+      
+      // Call LLM with both screenshots
+      const request = {
+        model: 'gpt-5-mini', // Vision model
+        systemPrompt: 'You are a visual verification expert for web automation. Compare before/after screenshots to determine if an action achieved its expected outcome.',
+        userPrompt: `Expected change: ${expectedChange}
+
+Compare the [BEFORE] and [AFTER] screenshots.
+
+Did the expected change occur? Respond with JSON:
+{
+  "verified": boolean,
+  "reasoning": "What changed (or didn't change) between screenshots",
+  "changes": ["Specific UI changes observed"]
+}
+
+Be objective:
+- Look for new elements, panels, forms, modals
+- Check for page navigation, URL changes, content changes
+- Note any visual indicators of success/failure
+
+Be strict: Only return true if you clearly see the expected change.`,
+        images: [
+          { label: 'BEFORE', dataUrl: beforeDataUrl },
+          { label: 'AFTER', dataUrl: afterDataUrl }
+        ]
+      };
+      
+      const response = await this.llmFacade.llmProvider.callLLM(request);
+      
+      // Parse response
+      const jsonMatch = response.answer.match(/\{[\s\S]*\}/);
+      if (!jsonMatch) {
+        logger?.(`[VerifyActionResult] ⚠️ Could not parse verification response`, 'warn');
+        return {
+          success: false,
+          error: 'Failed to parse verification response'
+        };
+      }
+      
+      const parsed = JSON.parse(jsonMatch[0]);
+      
+      logger?.(`[VerifyActionResult] 📊 Result: ${parsed.verified ? '✅ VERIFIED' : '❌ NOT VERIFIED'}`);
+      logger?.(`[VerifyActionResult] 💭 ${parsed.reasoning}`);
+      if (parsed.changes && parsed.changes.length > 0) {
+        logger?.(`[VerifyActionResult] 📝 Changes observed:`);
+        parsed.changes.forEach((change: string) => logger?.(`   - ${change}`));
+      }
+      
+      return {
+        success: true,
+        data: parsed,
+        learning: `Verification result: ${parsed.verified ? 'Goal achieved' : 'Goal not achieved'}. ${parsed.reasoning}`
+      };
+      
+    } catch (error: any) {
+      logger?.(`[VerifyActionResult] ❌ Error: ${error.message}`, 'error');
+      return {
+        success: false,
+        error: error.message
+      };
+    }
+  }
+}
+

package/src/orchestrator/tools/view-previous-screenshot.ts

@@ -0,0 +1,103 @@
+/**
+ * View Previous Screenshot Tool
+ * Access the screenshot from the previous iteration for continuity reasoning
+ */
+
+import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
+import { ToolResult } from '../types';
+
+export class ViewPreviousScreenshotTool implements Tool {
+  name = 'view_previous_screenshot';
+  description = 'View the screenshot from the PREVIOUS iteration to understand continuity. Common uses: (1) Verify coordinate-based clicks - look for magenta "clicked" marker to see where click landed, (2) Compare before/after states - see what changed after commands, (3) Check transient effects - see alerts/toasts that may have disappeared. Returns vision analysis of the previous screenshot.';
+  
+  // LLM facade for vision analysis
+  private llmFacade?: any;
+  
+  setLLMFacade(llmFacade: any): void {
+    this.llmFacade = llmFacade;
+  }
+  
+  parameters: ToolParameter[] = [
+    {
+      name: 'purpose',
+      type: 'string',
+      description: 'Why you need to see the previous screenshot. Examples: "verify coord click accuracy", "check if error message appeared then disappeared", "compare before/after form submission"',
+      required: true
+    }
+  ];
+  
+  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
+    const { logger } = context;
+    const purpose = params.purpose || 'Review previous page state';
+    
+    // Access previousSomScreenshot from context (passed through)
+    const previousScreenshot = (context as any).previousSomScreenshot;
+    
+    if (!previousScreenshot) {
+      return {
+        success: false,
+        error: 'No previous screenshot available (this is the first iteration)'
+      };
+    }
+    
+    try {
+      logger?.(`[ViewPreviousScreenshot] Analyzing previous iteration screenshot for: ${purpose}`, 'log');
+      
+      // Analyze with vision LLM if available
+      let analysis = 'Previous screenshot retrieved.';
+      
+      if (this.llmFacade) {
+        try {
+          const analysisPrompt = `Analyze the screenshot from the PREVIOUS iteration (before the most recent commands executed).
+
+PURPOSE: ${purpose}
+
+WHAT TO LOOK FOR:
+1. **Coordinate verification**: If purpose mentions "coord" or "click", look for MAGENTA "clicked" marker (circle with yellow border)
+   - Describe marker position relative to UI elements
+   - Assess accuracy: "centered on button", "5% above target", etc.
+
+2. **Before/after comparison**: Compare visual state with current page
+   - What changed after commands executed?
+   - New elements, removed elements, state changes?
+
+3. **Transient effects**: Elements that may have appeared and disappeared
+   - Alerts, toasts, error messages that are now gone
+   - Loading states, spinners that finished
+
+4. **General state**: Answer the specific question from purpose
+
+TASK: Provide concise, specific observations relevant to the purpose.`;
+
+          const llmResponse = await this.llmFacade.llmProvider.callLLM({
+            systemPrompt: 'You are analyzing a screenshot from a previous test iteration to help with continuity reasoning. Provide specific, actionable observations.',
+            userPrompt: analysisPrompt,
+            imageUrl: previousScreenshot
+          });
+          
+          analysis = llmResponse.answer || analysis;
+          logger?.(`[ViewPreviousScreenshot] ✓ Analysis complete`, 'log');
+          
+        } catch (error: any) {
+          logger?.(`[ViewPreviousScreenshot] ⚠ Vision analysis failed: ${error.message}`, 'warn');
+        }
+      }
+      
+      return {
+        success: true,
+        data: {
+          screenshotAvailable: true,
+          purpose
+        },
+        learning: analysis
+      };
+    } catch (error: any) {
+      logger?.(`[ViewPreviousScreenshot] ✗ Failed: ${error.message}`, 'error');
+      return {
+        success: false,
+        error: `Failed to access previous screenshot: ${error.message}`
+      };
+    }
+  }
+}
+