testchimp-runner-core 0.0.34 → 0.0.35

package/src/orchestrator/som-types.ts

@@ -0,0 +1,188 @@
+/**
+ * Set-of-Marks (SoM) Type Definitions
+ * Types for visual element identification and interaction
+ */
+
+export interface Coordinate {
+  x: number;  // Percentage of viewport width: 0-100 (use 3 decimal precision, e.g., 15.625)
+  y: number;  // Percentage of viewport height: 0-100 (use 3 decimal precision, e.g., 82.375)
+}
+
+export enum InteractionAction {
+  // Click actions
+  CLICK = 'click',
+  DOUBLE_CLICK = 'doubleClick',
+  RIGHT_CLICK = 'rightClick',
+  
+  // Mouse actions
+  HOVER = 'hover',
+  MOUSE_DOWN = 'mouseDown',
+  MOUSE_UP = 'mouseUp',
+  DRAG = 'drag',
+  
+  // Input actions
+  FILL = 'fill',
+  TYPE = 'type',
+  CLEAR = 'clear',
+  
+  // Keyboard actions
+  PRESS = 'press',
+  PRESS_SEQUENTIALLY = 'pressSequentially',
+  
+  // Select/Checkbox actions
+  SELECT = 'select',
+  CHECK = 'check',
+  UNCHECK = 'uncheck',
+  
+  // Focus/Scroll actions
+  FOCUS = 'focus',
+  BLUR = 'blur',
+  SCROLL = 'scroll',
+  SCROLL_INTO_VIEW = 'scrollIntoView',
+  
+  // Navigation actions
+  NAVIGATE = 'navigate',  // Go to URL (requires value field)
+  GO_BACK = 'goBack',
+  GO_FORWARD = 'goForward',
+  RELOAD = 'reload'
+}
+
+export interface SomCommand {
+  elementRef?: string;    // Integer as string: "1", "2", "42" (optional for coord-based commands)
+  action: InteractionAction;
+  
+  // Coordinate-based action (use when elementRef is empty/null)
+  coord?: Coordinate;     // Percentage-based (x: 0-100, y: 0-100 of viewport)
+  
+  // Action-specific parameters
+  value?: string;         // For fill/type/select/press actions
+  fromCoord?: Coordinate; // For drag (start) - percentage-based
+  toCoord?: Coordinate;   // For drag (end) - percentage-based
+  force?: boolean;        // Force action even if not actionable
+  scrollAmount?: number;  // Pixels to scroll
+  scrollDirection?: 'up' | 'down' | 'left' | 'right';
+  button?: 'left' | 'right' | 'middle';
+  clickCount?: number;
+  modifiers?: Array<'Alt' | 'Control' | 'Meta' | 'Shift'>;
+  delay?: number;         // Delay between keystrokes for TYPE (ms)
+  timeout?: number;       // Override default timeout
+}
+
+export enum CommandRunStatus {
+  SUCCESS = 'success',
+  FAILURE = 'failure'
+}
+
+export interface CommandAttempt {
+  command?: string;
+  status: CommandRunStatus;
+  error?: string;
+}
+
+export interface DomMutation {
+  type: 'added' | 'removed' | 'modified' | 'attribute_changed';
+  elementDescription: string;
+  timestamp: number;
+}
+
+export interface SemanticCommandResult {
+  failedAttempts: CommandAttempt[];
+  successAttempt?: CommandAttempt;
+  error?: string;
+  status: CommandRunStatus;
+  mutations?: DomMutation[];  // Only for hover/focus, filtered for relevance
+}
+
+export interface SomElement {
+  somId: string;  // Simple integer as string: "1", "2", "3"
+  tag: string;
+  role: string;
+  text: string;
+  ariaLabel: string;
+  placeholder: string;
+  name: string;
+  type: string;
+  id: string;
+  className: string;
+  bbox: { x: number; y: number; width: number; height: number };
+  parent?: {
+    tag: string;
+    role: string;
+    className: string;
+    text: string;
+  };
+}
+
+/**
+ * Typed selector (no string parsing needed)
+ * Supports chaining: parent.child for scoped selectors
+ */
+export interface TypedSelector {
+  type: 'id' | 'testId' | 'label' | 'role' | 'placeholder' | 'text' | 'title' | 'altText' | 'name' | 'locator';
+  value: string;
+  roleOptions?: { name?: string };  // For getByRole
+  parent?: TypedSelector;  // For chaining: page.locator(parent).locator(this)
+}
+
+/**
+ * Verification types for expect assertions
+ */
+export enum VerificationType {
+  // Text verifications
+  TEXT_CONTAINS = 'textContains',
+  TEXT_EQUALS = 'textEquals',
+  
+  // Input verifications
+  VALUE_EQUALS = 'valueEquals',
+  VALUE_EMPTY = 'valueEmpty',
+  
+  // Visibility verifications
+  IS_VISIBLE = 'isVisible',
+  IS_HIDDEN = 'isHidden',
+  
+  // State verifications
+  IS_ENABLED = 'isEnabled',
+  IS_DISABLED = 'isDisabled',
+  IS_CHECKED = 'isChecked',
+  IS_UNCHECKED = 'isUnchecked',
+  
+  // Count verifications (for lists, tables, etc.)
+  COUNT_EQUALS = 'countEquals',
+  COUNT_GREATER_THAN = 'countGreaterThan',
+  COUNT_LESS_THAN = 'countLessThan',
+  
+  // Attribute verifications
+  HAS_CLASS = 'hasClass',
+  HAS_ATTRIBUTE = 'hasAttribute'
+}
+
+/**
+ * SoM verification command for expect assertions
+ */
+export interface SomVerification {
+  verificationType: VerificationType;
+  elementRef?: string;       // SoM ID (e.g., "3") - optional for count verifications
+  expected?: string | number;  // Expected value/text/count
+  description?: string;        // Human-readable description
+  selector?: string;          // For count verifications on non-SoM elements (CSS selector)
+}
+
+/**
+ * Union type: commands array can contain both actions and verifications
+ */
+export type SomCommandOrVerification = SomCommand | SomVerification;
+
+/**
+ * Type guard to check if command is a verification
+ */
+export function isSomVerification(cmd: SomCommandOrVerification): cmd is SomVerification {
+  return 'verificationType' in cmd;
+}
+
+/**
+ * Type guard to check if command is an action
+ */
+export function isSomCommand(cmd: SomCommandOrVerification): cmd is SomCommand {
+  return 'action' in cmd;
+}
+

package/src/orchestrator/tool-registry.ts

@@ -40,6 +40,8 @@ export interface ToolExecutionContext {
   memory: any;  // JourneyMemory
   stepNumber: number;
   logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
+  previousSomScreenshot?: string;  // For view_previous_screenshot tool
+  somHandler?: any;  // PageSoMHandler for refresh_som_markers tool
 }
 
 /**

package/src/orchestrator/tools/index.ts

@@ -1,9 +1,12 @@
 /**
- * Tool exports - 6 information-gathering tools
+ * Tool exports - 8 information-gathering tools
  * Note: State changes (navigation, clicks, fills) are done via Playwright commands, not tools
+ * Ref-based commands (getByRef) are translated to Playwright at execution time
  */
 
 export { TakeScreenshotTool } from './take-screenshot';
+export { ViewPreviousScreenshotTool } from './view-previous-screenshot';
+export { RefreshSomMarkersTool } from './refresh-som-markers';
 export { RecallHistoryTool } from './recall-history';
 export { InspectPageTool } from './inspect-page';
 export { CheckPageReadyTool } from './check-page-ready';

package/src/orchestrator/tools/refresh-som-markers.ts

@@ -0,0 +1,69 @@
+import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
+import { ToolResult } from '../types';
+
+/**
+ * Tool to refresh SoM markers when they appear outdated or misaligned
+ */
+export class RefreshSomMarkersTool implements Tool {
+  name = 'refresh_som_markers';
+  description = 'Manually refresh the Set-of-Marks visual markers on the page. Use when: (1) Markers appear misaligned with actual UI elements, (2) Page content has changed but markers are stale (e.g., after dynamic content loads), (3) You suspect markers are from a previous page state. Returns updated screenshot with fresh markers.';
+  
+  parameters: ToolParameter[] = [
+    {
+      name: 'reason',
+      type: 'string',
+      description: 'Why you need to refresh markers. Examples: "dropdown expanded but markers still show closed state", "new content loaded but not marked", "markers seem to point to wrong elements"',
+      required: true
+    }
+  ];
+  
+  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
+    const { logger, page, somHandler } = context;
+    const reason = params.reason || 'Markers appear outdated';
+    
+    if (!somHandler) {
+      return {
+        success: false,
+        error: 'SoM mode not enabled - refresh markers tool unavailable.'
+      };
+    }
+    
+    if (!page) {
+      return {
+        success: false,
+        error: 'No page context available.'
+      };
+    }
+    
+    try {
+      logger?.(`[RefreshSomMarkers] Refreshing markers due to: ${reason}`, 'log');
+      
+      // Ensure somHandler has the latest page reference
+      somHandler.setPage(page);
+      
+      // Re-scan page and update markers
+      await somHandler.updateSom();
+      logger?.(`[RefreshSomMarkers] ✓ Markers updated`, 'log');
+      
+      // Capture fresh screenshot with new markers (viewport only - cheaper than full page)
+      const freshScreenshot = await somHandler.getScreenshot(true, false, 60);
+      logger?.(`[RefreshSomMarkers] ✓ Fresh screenshot captured (viewport)`, 'log');
+      
+      return {
+        success: true,
+        data: {
+          screenshot: freshScreenshot,
+          reason
+        },
+        learning: `SoM markers refreshed. New screenshot shows current page state with updated element markers. Reason: ${reason}`
+      };
+    } catch (error: any) {
+      logger?.(`[RefreshSomMarkers] ✗ Failed: ${error.message}`, 'error');
+      return {
+        success: false,
+        error: `Failed to refresh markers: ${error.message}`
+      };
+    }
+  }
+}
+

package/src/orchestrator/tools/view-previous-screenshot.ts

@@ -0,0 +1,103 @@
+/**
+ * View Previous Screenshot Tool
+ * Access the screenshot from the previous iteration for continuity reasoning
+ */
+
+import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
+import { ToolResult } from '../types';
+
+export class ViewPreviousScreenshotTool implements Tool {
+  name = 'view_previous_screenshot';
+  description = 'View the screenshot from the PREVIOUS iteration to understand continuity. Common uses: (1) Verify coordinate-based clicks - look for magenta "clicked" marker to see where click landed, (2) Compare before/after states - see what changed after commands, (3) Check transient effects - see alerts/toasts that may have disappeared. Returns vision analysis of the previous screenshot.';
+  
+  // LLM facade for vision analysis
+  private llmFacade?: any;
+  
+  setLLMFacade(llmFacade: any): void {
+    this.llmFacade = llmFacade;
+  }
+  
+  parameters: ToolParameter[] = [
+    {
+      name: 'purpose',
+      type: 'string',
+      description: 'Why you need to see the previous screenshot. Examples: "verify coord click accuracy", "check if error message appeared then disappeared", "compare before/after form submission"',
+      required: true
+    }
+  ];
+  
+  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
+    const { logger } = context;
+    const purpose = params.purpose || 'Review previous page state';
+    
+    // Access previousSomScreenshot from context (passed through)
+    const previousScreenshot = (context as any).previousSomScreenshot;
+    
+    if (!previousScreenshot) {
+      return {
+        success: false,
+        error: 'No previous screenshot available (this is the first iteration)'
+      };
+    }
+    
+    try {
+      logger?.(`[ViewPreviousScreenshot] Analyzing previous iteration screenshot for: ${purpose}`, 'log');
+      
+      // Analyze with vision LLM if available
+      let analysis = 'Previous screenshot retrieved.';
+      
+      if (this.llmFacade) {
+        try {
+          const analysisPrompt = `Analyze the screenshot from the PREVIOUS iteration (before the most recent commands executed).
+
+PURPOSE: ${purpose}
+
+WHAT TO LOOK FOR:
+1. **Coordinate verification**: If purpose mentions "coord" or "click", look for MAGENTA "clicked" marker (circle with yellow border)
+   - Describe marker position relative to UI elements
+   - Assess accuracy: "centered on button", "5% above target", etc.
+
+2. **Before/after comparison**: Compare visual state with current page
+   - What changed after commands executed?
+   - New elements, removed elements, state changes?
+
+3. **Transient effects**: Elements that may have appeared and disappeared
+   - Alerts, toasts, error messages that are now gone
+   - Loading states, spinners that finished
+
+4. **General state**: Answer the specific question from purpose
+
+TASK: Provide concise, specific observations relevant to the purpose.`;
+
+          const llmResponse = await this.llmFacade.llmProvider.callLLM({
+            systemPrompt: 'You are analyzing a screenshot from a previous test iteration to help with continuity reasoning. Provide specific, actionable observations.',
+            userPrompt: analysisPrompt,
+            imageUrl: previousScreenshot
+          });
+          
+          analysis = llmResponse.answer || analysis;
+          logger?.(`[ViewPreviousScreenshot] ✓ Analysis complete`, 'log');
+          
+        } catch (error: any) {
+          logger?.(`[ViewPreviousScreenshot] ⚠ Vision analysis failed: ${error.message}`, 'warn');
+        }
+      }
+      
+      return {
+        success: true,
+        data: {
+          screenshotAvailable: true,
+          purpose
+        },
+        learning: analysis
+      };
+    } catch (error: any) {
+      logger?.(`[ViewPreviousScreenshot] ✗ Failed: ${error.message}`, 'error');
+      return {
+        success: false,
+        error: `Failed to access previous screenshot: ${error.message}`
+      };
+    }
+  }
+}
+

package/src/orchestrator/types.ts

@@ -108,7 +108,7 @@ export interface AgentDecision {
   needsToolResults?: boolean;  // Wait for tool results before proceeding with commands
   
   // Command batch (executed sequentially)
-  commands?: string[];
+  commands?: string[];  // Plain Playwright commands
   commandReasoning?: string;
   
   // Self-reflection for next iteration
@@ -147,6 +147,12 @@ export interface AgentDecision {
     issue: 'prior_incomplete' | 'already_done' | 'wrong_order' | null;
     explanation: string;                   // Why agent thinks step order is off
   };
+  
+  // Meta-learning: Suggested prompt improvements based on journey learnings
+  debugInfo?: {
+    suggestedPromptUpdates?: string;      // Confident suggestions for improving system/user prompts
+    reasoning?: string;                    // Why these updates would help
+  };
 }
 
 /**
@@ -172,16 +178,34 @@ export interface AgentContext {
   experiences: string[];
   extractedData: Record<string, string>;
   
-  // Self-reflection from previous iteration
-  previousIterationGuidance?: SelfReflection;
-  
-  // Note from previous iteration (NEW - tactical continuity)
+  // Note from previous iteration (tactical continuity)
   noteFromPreviousIteration?: NoteToFutureSelf;
   
+  // Test data / credentials for exploration
+  testDataPrompt?: string;
+  
+  // SoM (Set-of-Marks) screenshot with visual markers
+  somScreenshot?: string;  // Data URL of screenshot with SoM markers
+  somElementMap?: string;  // Text map of SoM IDs to element details for disambiguation
+  
+  // Repair mode context (undefined for script gen/exploration)
+  priorSteps?: string[];   // Steps completed before current (e.g., ["1. Navigate", "2. Login"])
+  nextSteps?: string[];    // Steps after current (e.g., ["5. Submit", "6. Verify"])
+  
   // Tool results from this iteration (if any)
   toolResults?: Record<string, ToolResult>;
 }
 
+/**
+ * Exploration mode configuration
+ */
+export interface ExplorationMode {
+  enabled: boolean;                    // Whether exploration mode is active
+  explorationPrompt: string;           // Journey-specific focus: "Explore Dashboard and test all widgets"
+  testDataPrompt?: string;             // Test data, credentials context
+  maxExplorationSteps?: number;        // Budget limit (default: 50) - agent can stop earlier
+}
+
 /**
  * Configurable guardrails
  */
@@ -211,6 +235,15 @@ export interface AgentConfig {
   // Allowed actions
   allowedExplorationActions?: string[];   // Default: ['hover', 'click_info', 'click_menu', 'focus'] (Phase 2)
   allowedDomains?: string[];              // For navigate_to_url validation
+  
+  // Feature flags
+  enableCoordinateMode?: boolean;         // Default: false (experimental - disable until stable)
+  useSoM?: boolean;                       // Default: true (Set-of-Marks visual mode)
+  somUseSomIdBasedCommands?: boolean;     // Default: false (use semantic selectors first)
+  somRestrictCoordinates?: boolean;       // Default: false (if true, strongly discourage coord commands except as absolute last resort)
+  
+  // Exploration mode (NEW)
+  explorationMode?: ExplorationMode;
 }
 
 /**
@@ -243,6 +276,16 @@ export const DEFAULT_AGENT_CONFIG: Required<AgentConfig> = {
   commandTimeout: 30000,
   explorationTimeout: 2000,
   allowedExplorationActions: ['hover', 'click_info', 'click_menu', 'focus'],
-  allowedDomains: []
+  allowedDomains: [],
+  enableCoordinateMode: false,  // Disabled by default - experimental feature
+  useSoM: true,  // Enabled by default - use Set-of-Marks visual mode
+  somUseSomIdBasedCommands: false,  // Use semantic selectors first
+  somRestrictCoordinates: false,  // Allow coords as valid fallback (for exploration)
+  explorationMode: {
+    enabled: false,
+    explorationPrompt: '',
+    testDataPrompt: undefined,
+    maxExplorationSteps: 50
+  }
 };
 

package/src/scenario-service.ts

@@ -218,6 +218,26 @@ export class ScenarioService extends EventEmitter {
     this.processNextJob();
   }
 
+  /**
+   * Execute exploration mode using orchestrator
+   * Requires orchestrator to be enabled via useOrchestrator option
+   */
+  async executeExploration(page: any, explorationConfig: any, jobId: string): Promise<any> {
+    if (!this.useOrchestrator) {
+      throw new Error('Exploration mode requires orchestrator to be enabled');
+    }
+    
+    // Get an available worker (or create one if needed)
+    let worker = this.workers.find(w => !this.busyWorkers.has(w));
+    if (!worker) {
+      await this.createWorker();
+      worker = this.workers[this.workers.length - 1];
+    }
+    
+    // Execute exploration via worker's orchestrator
+    return worker.executeExploration(page, explorationConfig, jobId);
+  }
+
   async shutdown(): Promise<void> {
     this.log('Shutting down scenario service...');
     

package/src/scenario-worker-class.ts

@@ -18,6 +18,8 @@ import {
   JourneyMemory,
   AgentConfig,
   TakeScreenshotTool,
+  ViewPreviousScreenshotTool,
+  RefreshSomMarkersTool,
   RecallHistoryTool,
   InspectPageTool,
   CheckPageReadyTool,
@@ -105,11 +107,18 @@ export class ScenarioWorker extends EventEmitter {
     const takeScreenshotTool = new TakeScreenshotTool();
     takeScreenshotTool.setLLMFacade(this.llmFacade);  // Inject LLM for vision analysis
     
+    const viewPreviousScreenshotTool = new ViewPreviousScreenshotTool();
+    viewPreviousScreenshotTool.setLLMFacade(this.llmFacade);  // Inject LLM for vision analysis
+    
+    const refreshSomMarkersTool = new RefreshSomMarkersTool();
+    
     const verifyActionTool = new VerifyActionResultTool();
     verifyActionTool.setLLMFacade(this.llmFacade);  // Inject LLM for vision comparison
     
-    // Register 6 information-gathering tools (state changes via Playwright commands)
+    // Register 8 information-gathering tools (state changes via Playwright commands)
     this.toolRegistry.register(takeScreenshotTool);
+    this.toolRegistry.register(viewPreviousScreenshotTool);
+    this.toolRegistry.register(refreshSomMarkersTool);
     this.toolRegistry.register(new RecallHistoryTool());
     this.toolRegistry.register(new InspectPageTool());
     this.toolRegistry.register(new CheckPageReadyTool());
@@ -317,12 +326,12 @@ export class ScenarioWorker extends EventEmitter {
         page = job.existingPage;
       } else {
         // Create new browser (default behavior for local clients)
-        // Default to headed mode (headless: false) for better debugging
+        // Let the playwrightConfig control headless mode (don't override with hardcoded value)
         // Create logger function from outputChannel for browser initialization
         const logger = this.outputChannel ? (message: string, level?: 'log' | 'error' | 'warn') => {
           this.outputChannel!.appendLine(`[Browser] ${message}`);
         } : undefined;
-        const browserInstance = await initializeBrowser(job.playwrightConfig, false, undefined, logger);
+        const browserInstance = await initializeBrowser(job.playwrightConfig, undefined, undefined, logger);
         browser = browserInstance.browser;
         context = browserInstance.context;
         page = browserInstance.page;
@@ -1082,6 +1091,18 @@ export class ScenarioWorker extends EventEmitter {
 
 
 
+  /**
+   * Execute exploration mode using orchestrator
+   */
+  async executeExploration(page: any, explorationConfig: any, jobId: string): Promise<any> {
+    if (!this.useOrchestrator || !this.orchestratorAgent) {
+      throw new Error('Orchestrator not available - exploration mode requires orchestrator');
+    }
+    
+    // Execute exploration via orchestrator
+    return this.orchestratorAgent.executeExploration(page, explorationConfig, jobId);
+  }
+
   async cleanup(): Promise<void> {
     this.initialized = false;
     this.sessionId = null;

package/src/utils/page-info-retry.ts

@@ -0,0 +1,65 @@
+/**
+ * Page Info Retry Utility
+ * Handles adaptive page loading with exponential backoff
+ */
+
+import { getEnhancedPageInfo, PageInfo } from './page-info-utils';
+
+export class PageInfoRetry {
+  /**
+   * Get page info with retry logic - waits for interactive elements to appear
+   * Uses exponential backoff to handle slow-loading React/Vue/Angular apps
+   */
+  static async getWithRetry(page: any, maxAttempts: number = 6): Promise<PageInfo> {
+    // Wait for initial page load (generous timeout for slow apps)
+    try {
+      await page.waitForLoadState('domcontentloaded', { timeout: 20000 }).catch(() => {});
+    } catch (waitError) {
+      // Continue even if wait fails
+    }
+    
+    let attempt = 0;
+    let backoffMs = 1000; // Start with 1 second (adequate for most sites)
+    
+    while (attempt < maxAttempts) {
+      attempt++;
+      
+      // Try to extract page info
+      const pageInfo = await getEnhancedPageInfo(page);
+      
+      // If we got a reasonable number of elements, we're done
+      if (pageInfo.interactiveElements && pageInfo.interactiveElements.length >= 3) {
+        if (attempt > 1) {
+          console.log(`[PageInfoRetry] ✓ Page elements loaded after ${attempt} attempts`);
+        }
+        return pageInfo;
+      }
+      
+      // If this is the last attempt, return what we have
+      if (attempt >= maxAttempts) {
+        const totalWait = this.calculateTotalWaitTime(maxAttempts);
+        console.log(`[PageInfoRetry] ⚠️  Only found ${pageInfo.interactiveElements?.length || 0} elements after ${maxAttempts} attempts (total wait: ~${totalWait}ms)`);
+        return pageInfo;
+      }
+      
+      // Wait with exponential backoff before retrying
+      console.log(`[PageInfoRetry] Only ${pageInfo.interactiveElements?.length || 0} elements found (attempt ${attempt}/${maxAttempts}), waiting ${backoffMs}ms...`);
+      await page.waitForTimeout(backoffMs);
+      backoffMs = Math.min(backoffMs * 1.6, 15000); // Cap at 15 seconds per attempt
+    }
+    
+    // Fallback (shouldn't reach here, but for type safety)
+    return await getEnhancedPageInfo(page);
+  }
+  
+  private static calculateTotalWaitTime(maxAttempts: number): number {
+    let total = 0;
+    let backoffMs = 1000;
+    for (let i = 1; i < maxAttempts; i++) {
+      total += backoffMs;
+      backoffMs = Math.min(backoffMs * 1.6, 15000);
+    }
+    return Math.round(total);
+  }
+}
+