testchimp-runner-core 0.0.35 → 0.0.36

package/src/orchestrator/tools/refresh-som-markers.ts

@@ -1,69 +0,0 @@
-import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
-import { ToolResult } from '../types';
-
-/**
- * Tool to refresh SoM markers when they appear outdated or misaligned
- */
-export class RefreshSomMarkersTool implements Tool {
-  name = 'refresh_som_markers';
-  description = 'Manually refresh the Set-of-Marks visual markers on the page. Use when: (1) Markers appear misaligned with actual UI elements, (2) Page content has changed but markers are stale (e.g., after dynamic content loads), (3) You suspect markers are from a previous page state. Returns updated screenshot with fresh markers.';
-  
-  parameters: ToolParameter[] = [
-    {
-      name: 'reason',
-      type: 'string',
-      description: 'Why you need to refresh markers. Examples: "dropdown expanded but markers still show closed state", "new content loaded but not marked", "markers seem to point to wrong elements"',
-      required: true
-    }
-  ];
-  
-  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
-    const { logger, page, somHandler } = context;
-    const reason = params.reason || 'Markers appear outdated';
-    
-    if (!somHandler) {
-      return {
-        success: false,
-        error: 'SoM mode not enabled - refresh markers tool unavailable.'
-      };
-    }
-    
-    if (!page) {
-      return {
-        success: false,
-        error: 'No page context available.'
-      };
-    }
-    
-    try {
-      logger?.(`[RefreshSomMarkers] Refreshing markers due to: ${reason}`, 'log');
-      
-      // Ensure somHandler has the latest page reference
-      somHandler.setPage(page);
-      
-      // Re-scan page and update markers
-      await somHandler.updateSom();
-      logger?.(`[RefreshSomMarkers] ✓ Markers updated`, 'log');
-      
-      // Capture fresh screenshot with new markers (viewport only - cheaper than full page)
-      const freshScreenshot = await somHandler.getScreenshot(true, false, 60);
-      logger?.(`[RefreshSomMarkers] ✓ Fresh screenshot captured (viewport)`, 'log');
-      
-      return {
-        success: true,
-        data: {
-          screenshot: freshScreenshot,
-          reason
-        },
-        learning: `SoM markers refreshed. New screenshot shows current page state with updated element markers. Reason: ${reason}`
-      };
-    } catch (error: any) {
-      logger?.(`[RefreshSomMarkers] ✗ Failed: ${error.message}`, 'error');
-      return {
-        success: false,
-        error: `Failed to refresh markers: ${error.message}`
-      };
-    }
-  }
-}
-

package/src/orchestrator/tools/take-screenshot.ts

@@ -1,128 +0,0 @@
-/**
- * Take Screenshot Tool
- * Captures current page state visually and analyzes it with DOM snapshot
- */
-
-import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
-import { ToolResult } from '../types';
-import { getEnhancedPageInfo } from '../../utils/page-info-utils';
-import { VISION_MODEL } from '../../model-constants';
-
-export class TakeScreenshotTool implements Tool {
-  name = 'take_screenshot';
-  description = 'Capture a screenshot and analyze it with DOM snapshot to get actionable selector recommendations. Use when you need to see the actual page visually to find the right elements. Returns text-based analysis with selector suggestions.';
-  
-  // LLM facade will be injected
-  private llmFacade?: any;
-  
-  setLLMFacade(llmFacade: any): void {
-    this.llmFacade = llmFacade;
-  }
-  
-  parameters: ToolParameter[] = [
-    {
-      name: 'isFullPage',
-      type: 'boolean',
-      description: 'If true, captures entire page (scrolling). If false, captures only viewport. DEFAULT: true (recommended for finding elements that may be below fold)',
-      required: false,
-      default: true
-    },
-    {
-      name: 'purpose',
-      type: 'string',
-      description: 'What you are trying to find or understand from the screenshot',
-      required: false
-    }
-  ];
-  
-  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
-    const { page, logger } = context;
-    const isFullPage = params.isFullPage !== undefined ? params.isFullPage : true;  // Default: true (capture full page)
-    const purpose = params.purpose || 'Analyze page structure and elements';
-    
-    try {
-      logger?.(`[TakeScreenshot] Capturing ${isFullPage ? 'full page' : 'viewport'} screenshot for: ${purpose}`, 'log');
-      
-      // Capture screenshot (JPEG 60 quality for smaller size)
-      const screenshotBuffer = await page.screenshot({
-        fullPage: isFullPage,
-        type: 'jpeg',
-        quality: 60
-      });
-      
-      // Convert to data URL
-      const base64 = screenshotBuffer.toString('base64');
-      const dataUrl = `data:image/jpeg;base64,${base64}`;
-      
-      logger?.(`[TakeScreenshot] ✓ Screenshot captured (${Math.round(base64.length / 1024)}KB), analyzing with DOM...`, 'log');
-      
-      // Get DOM snapshot for correlation (already has ARIA tree + interactive elements with bboxes)
-      const pageInfo = await getEnhancedPageInfo(page);
-      
-      // Analyze screenshot with structured DOM via LLM
-      let analysis = 'Screenshot captured. Use DOM snapshot in context to find selectors.';
-      
-      if (this.llmFacade) {
-        try {
-          logger?.(`[TakeScreenshot] Calling LLM for vision analysis...`, 'log');
-          
-          const analysisPrompt = `Analyze screenshot WITH DOM structure to identify correct Playwright selectors.
-
-PURPOSE: ${purpose}
-
-INTERACTIVE ELEMENTS (with positions and suggested selectors):
-${pageInfo.formattedElements}
-
-ARIA TREE (hierarchical structure):
-${JSON.stringify(pageInfo.ariaSnapshot, null, 2).substring(0, 2000)}
-
-TASK:
-1. Look at screenshot - identify the visual elements you need for: ${purpose}
-2. Match visual position with bounding boxes above
-3. Recommend SEMANTIC SELECTORS FIRST: getByRole, getByLabel, getByPlaceholder, getByText
-4. AVOID auto-generated IDs with unicode (e.g., #«r3»-form-item)
-
-Output format:
-"For [visual element description]:
- Try: [semantic selector from list - prefer getByRole/getByLabel]
- Or: [alternative selector]"
-
-Be concise. Only 2-3 recommendations. Prioritize user-friendly semantic selectors.`;
-
-
-          const llmResponse = await this.llmFacade.llmProvider.callLLM({
-            model: VISION_MODEL,
-            systemPrompt: 'You are a vision analysis expert for web automation. Analyze screenshots with DOM snapshots to recommend working Playwright selectors. ALWAYS prioritize semantic selectors (getByRole, getByLabel, getByText) over CSS selectors with auto-generated IDs.',
-            userPrompt: analysisPrompt,
-            imageUrl: dataUrl
-          });
-          
-          analysis = llmResponse.answer || analysis;
-          logger?.(`[TakeScreenshot] ✓ Vision analysis complete`, 'log');
-          
-        } catch (error: any) {
-          logger?.(`[TakeScreenshot] ⚠ Vision analysis failed, returning raw screenshot: ${error.message}`, 'warn');
-        }
-      }
-      
-      return {
-        success: true,
-        data: {
-          screenshot: dataUrl,
-          isFullPage,
-          size: base64.length,
-          interactiveElementCount: pageInfo.interactiveElements.length
-        },
-        learning: analysis  // Text-based analysis with selector recommendations
-      };
-    } catch (error: any) {
-      logger?.(`[TakeScreenshot] ✗ Failed: ${error.message}`, 'error');
-      return {
-        success: false,
-        error: `Screenshot capture failed: ${error.message}`
-      };
-    }
-  }
-}
-
-

package/src/orchestrator/tools/verify-action-result.ts

@@ -1,159 +0,0 @@
-/**
- * Verify Action Result Tool
- * 
- * Compares before/after screenshots to verify if an action achieved its goal
- * USE SPARINGLY - expensive (sends 2 images to vision model)
- */
-
-import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
-import { ToolResult } from '../types';
-import { LabeledImage } from '../../llm-provider';
-
-export class VerifyActionResultTool implements Tool {
-  name = 'verify_action_result';
-  
-  description = `Verify if your previous action achieved its goal by comparing before/after screenshots.
-
-⚠️ USE SPARINGLY - EXPENSIVE (2 images sent to vision model)
-
-When to use:
-- After coordinate click to verify expected UI change occurred
-- After clicking element when unsure if goal achieved (no clear feedback)
-- To verify navigation or modal appeared
-
-How it works:
-1. Takes screenshot BEFORE you run your next command
-2. You run command (click, fill, etc.)  
-3. Takes screenshot AFTER command executes
-4. Compares both and tells you if expected change happened
-
-Parameters:
-- expectedChange: What should have changed (e.g., "Dashboard page loaded", "Modal appeared", "Form submitted")
-
-Returns: { verified: boolean, reasoning: string, changes: string[] }
-
-Example:
-{
-  "toolCalls": [{"name": "verify_action_result", "params": {"expectedChange": "Dashboard page loaded with data grid"}}],
-  "toolReasoning": "Need to verify coordinate click navigated to correct page",
-  "needsToolResults": true  // IMPORTANT: Wait for verification before deciding status
-}`;
-
-  parameters: ToolParameter[] = [
-    {
-      name: 'expectedChange',
-      type: 'string',
-      description: 'What UI change you expect to see (e.g., "Modal opened", "Page navigated", "New form appeared")',
-      required: true
-    }
-  ];
-  
-  // LLM facade will be injected
-  private llmFacade?: any;
-  
-  setLLMFacade(llmFacade: any): void {
-    this.llmFacade = llmFacade;
-  }
-
-  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
-    const { page, logger } = context;
-    const expectedChange = params.expectedChange || 'Expected UI change';
-    
-    logger?.(`[VerifyActionResult] Capturing before/after screenshots...`);
-    
-    try {
-      // Capture BEFORE screenshot (viewport only, JPEG 60% quality for cost efficiency)
-      const beforeScreenshot = await page.screenshot({ 
-        encoding: 'base64', 
-        fullPage: false,
-        type: 'jpeg',
-        quality: 60
-      });
-      const beforeDataUrl = `data:image/jpeg;base64,${beforeScreenshot}`;
-      
-      logger?.(`[VerifyActionResult] ✓ BEFORE screenshot captured (JPEG 60%)`);
-      logger?.(`[VerifyActionResult] ⏳ Waiting 1.5s for UI to settle after your previous action...`);
-      
-      // Wait for UI to settle after previous action
-      await page.waitForTimeout(1500);
-      
-      // Capture AFTER screenshot
-      const afterScreenshot = await page.screenshot({ 
-        encoding: 'base64', 
-        fullPage: false,
-        type: 'jpeg',
-        quality: 60
-      });
-      const afterDataUrl = `data:image/jpeg;base64,${afterScreenshot}`;
-      
-      logger?.(`[VerifyActionResult] ✓ AFTER screenshot captured (JPEG 60%)`);
-      logger?.(`[VerifyActionResult] 🔍 Calling vision model to compare...`);
-      
-      if (!this.llmFacade) {
-        throw new Error('LLM facade not initialized');
-      }
-      
-      // Call LLM with both screenshots
-      const request = {
-        model: 'gpt-5-mini', // Vision model
-        systemPrompt: 'You are a visual verification expert for web automation. Compare before/after screenshots to determine if an action achieved its expected outcome.',
-        userPrompt: `Expected change: ${expectedChange}
-
-Compare the [BEFORE] and [AFTER] screenshots.
-
-Did the expected change occur? Respond with JSON:
-{
-  "verified": boolean,
-  "reasoning": "What changed (or didn't change) between screenshots",
-  "changes": ["Specific UI changes observed"]
-}
-
-Be objective:
-- Look for new elements, panels, forms, modals
-- Check for page navigation, URL changes, content changes
-- Note any visual indicators of success/failure
-
-Be strict: Only return true if you clearly see the expected change.`,
-        images: [
-          { label: 'BEFORE', dataUrl: beforeDataUrl },
-          { label: 'AFTER', dataUrl: afterDataUrl }
-        ]
-      };
-      
-      const response = await this.llmFacade.llmProvider.callLLM(request);
-      
-      // Parse response
-      const jsonMatch = response.answer.match(/\{[\s\S]*\}/);
-      if (!jsonMatch) {
-        logger?.(`[VerifyActionResult] ⚠️ Could not parse verification response`, 'warn');
-        return {
-          success: false,
-          error: 'Failed to parse verification response'
-        };
-      }
-      
-      const parsed = JSON.parse(jsonMatch[0]);
-      
-      logger?.(`[VerifyActionResult] 📊 Result: ${parsed.verified ? '✅ VERIFIED' : '❌ NOT VERIFIED'}`);
-      logger?.(`[VerifyActionResult] 💭 ${parsed.reasoning}`);
-      if (parsed.changes && parsed.changes.length > 0) {
-        logger?.(`[VerifyActionResult] 📝 Changes observed:`);
-        parsed.changes.forEach((change: string) => logger?.(`   - ${change}`));
-      }
-      
-      return {
-        success: true,
-        data: parsed,
-        learning: `Verification result: ${parsed.verified ? 'Goal achieved' : 'Goal not achieved'}. ${parsed.reasoning}`
-      };
-      
-    } catch (error: any) {
-      logger?.(`[VerifyActionResult] ❌ Error: ${error.message}`, 'error');
-      return {
-        success: false,
-        error: error.message
-      };
-    }
-  }
-}
-

package/src/orchestrator/tools/view-previous-screenshot.ts

@@ -1,103 +0,0 @@
-/**
- * View Previous Screenshot Tool
- * Access the screenshot from the previous iteration for continuity reasoning
- */
-
-import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
-import { ToolResult } from '../types';
-
-export class ViewPreviousScreenshotTool implements Tool {
-  name = 'view_previous_screenshot';
-  description = 'View the screenshot from the PREVIOUS iteration to understand continuity. Common uses: (1) Verify coordinate-based clicks - look for magenta "clicked" marker to see where click landed, (2) Compare before/after states - see what changed after commands, (3) Check transient effects - see alerts/toasts that may have disappeared. Returns vision analysis of the previous screenshot.';
-  
-  // LLM facade for vision analysis
-  private llmFacade?: any;
-  
-  setLLMFacade(llmFacade: any): void {
-    this.llmFacade = llmFacade;
-  }
-  
-  parameters: ToolParameter[] = [
-    {
-      name: 'purpose',
-      type: 'string',
-      description: 'Why you need to see the previous screenshot. Examples: "verify coord click accuracy", "check if error message appeared then disappeared", "compare before/after form submission"',
-      required: true
-    }
-  ];
-  
-  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
-    const { logger } = context;
-    const purpose = params.purpose || 'Review previous page state';
-    
-    // Access previousSomScreenshot from context (passed through)
-    const previousScreenshot = (context as any).previousSomScreenshot;
-    
-    if (!previousScreenshot) {
-      return {
-        success: false,
-        error: 'No previous screenshot available (this is the first iteration)'
-      };
-    }
-    
-    try {
-      logger?.(`[ViewPreviousScreenshot] Analyzing previous iteration screenshot for: ${purpose}`, 'log');
-      
-      // Analyze with vision LLM if available
-      let analysis = 'Previous screenshot retrieved.';
-      
-      if (this.llmFacade) {
-        try {
-          const analysisPrompt = `Analyze the screenshot from the PREVIOUS iteration (before the most recent commands executed).
-
-PURPOSE: ${purpose}
-
-WHAT TO LOOK FOR:
-1. **Coordinate verification**: If purpose mentions "coord" or "click", look for MAGENTA "clicked" marker (circle with yellow border)
-   - Describe marker position relative to UI elements
-   - Assess accuracy: "centered on button", "5% above target", etc.
-
-2. **Before/after comparison**: Compare visual state with current page
-   - What changed after commands executed?
-   - New elements, removed elements, state changes?
-
-3. **Transient effects**: Elements that may have appeared and disappeared
-   - Alerts, toasts, error messages that are now gone
-   - Loading states, spinners that finished
-
-4. **General state**: Answer the specific question from purpose
-
-TASK: Provide concise, specific observations relevant to the purpose.`;
-
-          const llmResponse = await this.llmFacade.llmProvider.callLLM({
-            systemPrompt: 'You are analyzing a screenshot from a previous test iteration to help with continuity reasoning. Provide specific, actionable observations.',
-            userPrompt: analysisPrompt,
-            imageUrl: previousScreenshot
-          });
-          
-          analysis = llmResponse.answer || analysis;
-          logger?.(`[ViewPreviousScreenshot] ✓ Analysis complete`, 'log');
-          
-        } catch (error: any) {
-          logger?.(`[ViewPreviousScreenshot] ⚠ Vision analysis failed: ${error.message}`, 'warn');
-        }
-      }
-      
-      return {
-        success: true,
-        data: {
-          screenshotAvailable: true,
-          purpose
-        },
-        learning: analysis
-      };
-    } catch (error: any) {
-      logger?.(`[ViewPreviousScreenshot] ✗ Failed: ${error.message}`, 'error');
-      return {
-        success: false,
-        error: `Failed to access previous screenshot: ${error.message}`
-      };
-    }
-  }
-}
-