testchimp-runner-core 0.0.34 → 0.0.36

package/src/orchestrator/tools/check-page-ready.ts

@@ -1,75 +0,0 @@
-/**
- * Check Page Ready Tool
- * Verify that page has finished loading and is interactive
- */
-
-import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
-import { ToolResult } from '../types';
-
-export class CheckPageReadyTool implements Tool {
-  name = 'check_page_ready';
-  description = 'Verify that the page has finished loading and is ready for interaction. Checks load state, no pending network requests, and DOM stability.';
-  
-  parameters: ToolParameter[] = [
-    {
-      name: 'timeout',
-      type: 'number',
-      description: 'Maximum time to wait for page to be ready (milliseconds)',
-      required: false,
-      default: 5000
-    }
-  ];
-  
-  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
-    const { page, logger } = context;
-    const timeout = params.timeout || 5000;
-    
-    try {
-      logger?.(`[CheckPageReady] Verifying page is ready (timeout: ${timeout}ms)`, 'log');
-      
-      // Wait for load state
-      await page.waitForLoadState('domcontentloaded', { timeout });
-      
-      // Check if page is interactive (use string to avoid TypeScript DOM type issues)
-      const isInteractive = await page.evaluate(() => {
-        const doc = (globalThis as any).document;
-        return doc.readyState === 'complete' || doc.readyState === 'interactive';
-      });
-      
-      // Get page state info
-      const url = page.url();
-      const title = await page.title();
-      
-      if (isInteractive) {
-        logger?.(`[CheckPageReady] ✓ Page is ready: ${title} (${url})`, 'log');
-        return {
-          success: true,
-          data: {
-            ready: true,
-            url,
-            title,
-            readyState: 'complete'
-          }
-        };
-      } else {
-        logger?.(`[CheckPageReady] ⚠ Page not fully ready yet`, 'warn');
-        return {
-          success: true,
-          data: {
-            ready: false,
-            url,
-            title,
-            readyState: 'loading'
-          }
-        };
-      }
-    } catch (error: any) {
-      logger?.(`[CheckPageReady] ✗ Failed: ${error.message}`, 'error');
-      return {
-        success: false,
-        error: `Page ready check failed: ${error.message}`
-      };
-    }
-  }
-}
-

package/src/orchestrator/tools/extract-data.ts

@@ -1,92 +0,0 @@
-/**
- * Extract Data Tool
- * Save data from page for use in later steps
- */
-
-import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
-import { ToolResult } from '../types';
-import { JourneyMemory } from '../types';
-
-export class ExtractDataTool implements Tool {
-  name = 'extract_data';
-  description = 'Extract and save data from the page for use in later steps. Useful for capturing IDs, usernames, confirmation codes, etc.';
-  
-  parameters: ToolParameter[] = [
-    {
-      name: 'selector',
-      type: 'string',
-      description: 'CSS selector or text content to extract',
-      required: true
-    },
-    {
-      name: 'dataName',
-      type: 'string',
-      description: 'Name to save the data under (e.g., "userId", "confirmationCode")',
-      required: true
-    },
-    {
-      name: 'attribute',
-      type: 'string',
-      description: 'Optional: Extract specific attribute instead of text content (e.g., "href", "value")',
-      required: false
-    }
-  ];
-  
-  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
-    const { page, memory, logger } = context;
-    const journeyMemory = memory as JourneyMemory;
-    const { selector, dataName, attribute } = params;
-    
-    try {
-      logger?.(`[ExtractData] Extracting "${dataName}" from selector: ${selector}`, 'log');
-      
-      // Try to find element
-      const element = await page.locator(selector).first();
-      const count = await page.locator(selector).count();
-      
-      if (count === 0) {
-        logger?.(`[ExtractData] ✗ Element not found: ${selector}`, 'error');
-        return {
-          success: false,
-          error: `Element not found: ${selector}`
-        };
-      }
-      
-      // Extract data
-      let extractedValue: string;
-      if (attribute) {
-        extractedValue = await element.getAttribute(attribute) || '';
-      } else {
-        extractedValue = await element.textContent() || '';
-      }
-      
-      extractedValue = extractedValue.trim();
-      
-      if (!extractedValue) {
-        logger?.(`[ExtractData] ⚠ Extracted empty value from ${selector}`, 'warn');
-      }
-      
-      // Save to memory
-      journeyMemory.extractedData[dataName] = extractedValue;
-      
-      logger?.(`[ExtractData] ✓ Saved "${dataName}" = "${extractedValue.substring(0, 50)}${extractedValue.length > 50 ? '...' : ''}"`, 'log');
-      
-      return {
-        success: true,
-        data: {
-          dataName,
-          value: extractedValue,
-          selector,
-          attribute
-        }
-      };
-    } catch (error: any) {
-      logger?.(`[ExtractData] ✗ Failed: ${error.message}`, 'error');
-      return {
-        success: false,
-        error: `Data extraction failed: ${error.message}`
-      };
-    }
-  }
-}
-

package/src/orchestrator/tools/index.ts

@@ -1,12 +0,0 @@
-/**
- * Tool exports - 6 information-gathering tools
- * Note: State changes (navigation, clicks, fills) are done via Playwright commands, not tools
- */
-
-export { TakeScreenshotTool } from './take-screenshot';
-export { RecallHistoryTool } from './recall-history';
-export { InspectPageTool } from './inspect-page';
-export { CheckPageReadyTool } from './check-page-ready';
-export { ExtractDataTool } from './extract-data';
-export { VerifyActionResultTool } from './verify-action-result';
-

package/src/orchestrator/tools/inspect-page.ts

@@ -1,42 +0,0 @@
-/**
- * Inspect Page Tool
- * Get current DOM snapshot (might be redundant since DOM is always-provided, but keeps extensibility)
- */
-
-import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
-import { ToolResult } from '../types';
-import { getEnhancedPageInfo } from '../../utils/page-info-utils';
-
-export class InspectPageTool implements Tool {
-  name = 'inspect_page';
-  description = 'Get current page DOM snapshot. Note: Current page info is already provided in context, so this tool is mainly for forcing a fresh snapshot if needed.';
-  
-  parameters: ToolParameter[] = [];
-  
-  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
-    const { page, logger } = context;
-    
-    try {
-      logger?.(`[InspectPage] Getting fresh DOM snapshot`, 'log');
-      
-      const pageInfo = await getEnhancedPageInfo(page);
-      
-      logger?.(`[InspectPage] ✓ DOM snapshot retrieved`, 'log');
-      
-      return {
-        success: true,
-        data: {
-          pageInfo,
-          url: page.url()
-        }
-      };
-    } catch (error: any) {
-      logger?.(`[InspectPage] ✗ Failed: ${error.message}`, 'error');
-      return {
-        success: false,
-        error: `Page inspection failed: ${error.message}`
-      };
-    }
-  }
-}
-

package/src/orchestrator/tools/recall-history.ts

@@ -1,72 +0,0 @@
-/**
- * Recall History Tool
- * Access deeper history beyond the recent 6-7 steps always provided
- */
-
-import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
-import { ToolResult } from '../types';
-import { JourneyMemory } from '../types';
-
-export class RecallHistoryTool implements Tool {
-  name = 'recall_history';
-  description = 'Access journey history beyond the recent 6-7 steps always provided in context. Use when you need to remember what happened earlier in the journey.';
-  
-  parameters: ToolParameter[] = [
-    {
-      name: 'maxSteps',
-      type: 'number',
-      description: 'Maximum number of historical steps to retrieve (from most recent backwards)',
-      required: false,
-      default: 10
-    },
-    {
-      name: 'query',
-      type: 'string',
-      description: 'Optional: Filter for specific actions (e.g., "login", "form fill")',
-      required: false
-    }
-  ];
-  
-  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
-    const { memory, logger } = context;
-    const journeyMemory = memory as JourneyMemory;
-    const maxSteps = params.maxSteps || 10;
-    const query = params.query?.toLowerCase();
-    
-    try {
-      logger?.(`[RecallHistory] Retrieving up to ${maxSteps} historical steps${query ? ` matching "${query}"` : ''}`, 'log');
-      
-      let steps = [...journeyMemory.history];
-      
-      // Filter by query if provided
-      if (query) {
-        steps = steps.filter(step => 
-          step.action.toLowerCase().includes(query) ||
-          step.code.toLowerCase().includes(query) ||
-          step.observation.toLowerCase().includes(query)
-        );
-      }
-      
-      // Take most recent N steps
-      const recentSteps = steps.slice(-maxSteps);
-      
-      logger?.(`[RecallHistory] ✓ Found ${recentSteps.length} historical steps`, 'log');
-      
-      return {
-        success: true,
-        data: {
-          steps: recentSteps,
-          totalHistorySize: journeyMemory.history.length,
-          filtered: !!query
-        }
-      };
-    } catch (error: any) {
-      logger?.(`[RecallHistory] ✗ Failed: ${error.message}`, 'error');
-      return {
-        success: false,
-        error: `History recall failed: ${error.message}`
-      };
-    }
-  }
-}
-

package/src/orchestrator/tools/take-screenshot.ts

@@ -1,128 +0,0 @@
-/**
- * Take Screenshot Tool
- * Captures current page state visually and analyzes it with DOM snapshot
- */
-
-import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
-import { ToolResult } from '../types';
-import { getEnhancedPageInfo } from '../../utils/page-info-utils';
-import { VISION_MODEL } from '../../model-constants';
-
-export class TakeScreenshotTool implements Tool {
-  name = 'take_screenshot';
-  description = 'Capture a screenshot and analyze it with DOM snapshot to get actionable selector recommendations. Use when you need to see the actual page visually to find the right elements. Returns text-based analysis with selector suggestions.';
-  
-  // LLM facade will be injected
-  private llmFacade?: any;
-  
-  setLLMFacade(llmFacade: any): void {
-    this.llmFacade = llmFacade;
-  }
-  
-  parameters: ToolParameter[] = [
-    {
-      name: 'isFullPage',
-      type: 'boolean',
-      description: 'If true, captures entire page (scrolling). If false, captures only viewport. DEFAULT: true (recommended for finding elements that may be below fold)',
-      required: false,
-      default: true
-    },
-    {
-      name: 'purpose',
-      type: 'string',
-      description: 'What you are trying to find or understand from the screenshot',
-      required: false
-    }
-  ];
-  
-  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
-    const { page, logger } = context;
-    const isFullPage = params.isFullPage !== undefined ? params.isFullPage : true;  // Default: true (capture full page)
-    const purpose = params.purpose || 'Analyze page structure and elements';
-    
-    try {
-      logger?.(`[TakeScreenshot] Capturing ${isFullPage ? 'full page' : 'viewport'} screenshot for: ${purpose}`, 'log');
-      
-      // Capture screenshot (JPEG 60 quality for smaller size)
-      const screenshotBuffer = await page.screenshot({
-        fullPage: isFullPage,
-        type: 'jpeg',
-        quality: 60
-      });
-      
-      // Convert to data URL
-      const base64 = screenshotBuffer.toString('base64');
-      const dataUrl = `data:image/jpeg;base64,${base64}`;
-      
-      logger?.(`[TakeScreenshot] ✓ Screenshot captured (${Math.round(base64.length / 1024)}KB), analyzing with DOM...`, 'log');
-      
-      // Get DOM snapshot for correlation (already has ARIA tree + interactive elements with bboxes)
-      const pageInfo = await getEnhancedPageInfo(page);
-      
-      // Analyze screenshot with structured DOM via LLM
-      let analysis = 'Screenshot captured. Use DOM snapshot in context to find selectors.';
-      
-      if (this.llmFacade) {
-        try {
-          logger?.(`[TakeScreenshot] Calling LLM for vision analysis...`, 'log');
-          
-          const analysisPrompt = `Analyze screenshot WITH DOM structure to identify correct Playwright selectors.
-
-PURPOSE: ${purpose}
-
-INTERACTIVE ELEMENTS (with positions and suggested selectors):
-${pageInfo.formattedElements}
-
-ARIA TREE (hierarchical structure):
-${JSON.stringify(pageInfo.ariaSnapshot, null, 2).substring(0, 2000)}
-
-TASK:
-1. Look at screenshot - identify the visual elements you need for: ${purpose}
-2. Match visual position with bounding boxes above
-3. Recommend SEMANTIC SELECTORS FIRST: getByRole, getByLabel, getByPlaceholder, getByText
-4. AVOID auto-generated IDs with unicode (e.g., #«r3»-form-item)
-
-Output format:
-"For [visual element description]:
- Try: [semantic selector from list - prefer getByRole/getByLabel]
- Or: [alternative selector]"
-
-Be concise. Only 2-3 recommendations. Prioritize user-friendly semantic selectors.`;
-
-
-          const llmResponse = await this.llmFacade.llmProvider.callLLM({
-            model: VISION_MODEL,
-            systemPrompt: 'You are a vision analysis expert for web automation. Analyze screenshots with DOM snapshots to recommend working Playwright selectors. ALWAYS prioritize semantic selectors (getByRole, getByLabel, getByText) over CSS selectors with auto-generated IDs.',
-            userPrompt: analysisPrompt,
-            imageUrl: dataUrl
-          });
-          
-          analysis = llmResponse.answer || analysis;
-          logger?.(`[TakeScreenshot] ✓ Vision analysis complete`, 'log');
-          
-        } catch (error: any) {
-          logger?.(`[TakeScreenshot] ⚠ Vision analysis failed, returning raw screenshot: ${error.message}`, 'warn');
-        }
-      }
-      
-      return {
-        success: true,
-        data: {
-          screenshot: dataUrl,
-          isFullPage,
-          size: base64.length,
-          interactiveElementCount: pageInfo.interactiveElements.length
-        },
-        learning: analysis  // Text-based analysis with selector recommendations
-      };
-    } catch (error: any) {
-      logger?.(`[TakeScreenshot] ✗ Failed: ${error.message}`, 'error');
-      return {
-        success: false,
-        error: `Screenshot capture failed: ${error.message}`
-      };
-    }
-  }
-}
-
-

package/src/orchestrator/tools/verify-action-result.ts

@@ -1,159 +0,0 @@
-/**
- * Verify Action Result Tool
- * 
- * Compares before/after screenshots to verify if an action achieved its goal
- * USE SPARINGLY - expensive (sends 2 images to vision model)
- */
-
-import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
-import { ToolResult } from '../types';
-import { LabeledImage } from '../../llm-provider';
-
-export class VerifyActionResultTool implements Tool {
-  name = 'verify_action_result';
-  
-  description = `Verify if your previous action achieved its goal by comparing before/after screenshots.
-
-⚠️ USE SPARINGLY - EXPENSIVE (2 images sent to vision model)
-
-When to use:
-- After coordinate click to verify expected UI change occurred
-- After clicking element when unsure if goal achieved (no clear feedback)
-- To verify navigation or modal appeared
-
-How it works:
-1. Takes screenshot BEFORE you run your next command
-2. You run command (click, fill, etc.)  
-3. Takes screenshot AFTER command executes
-4. Compares both and tells you if expected change happened
-
-Parameters:
-- expectedChange: What should have changed (e.g., "Dashboard page loaded", "Modal appeared", "Form submitted")
-
-Returns: { verified: boolean, reasoning: string, changes: string[] }
-
-Example:
-{
-  "toolCalls": [{"name": "verify_action_result", "params": {"expectedChange": "Dashboard page loaded with data grid"}}],
-  "toolReasoning": "Need to verify coordinate click navigated to correct page",
-  "needsToolResults": true  // IMPORTANT: Wait for verification before deciding status
-}`;
-
-  parameters: ToolParameter[] = [
-    {
-      name: 'expectedChange',
-      type: 'string',
-      description: 'What UI change you expect to see (e.g., "Modal opened", "Page navigated", "New form appeared")',
-      required: true
-    }
-  ];
-  
-  // LLM facade will be injected
-  private llmFacade?: any;
-  
-  setLLMFacade(llmFacade: any): void {
-    this.llmFacade = llmFacade;
-  }
-
-  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
-    const { page, logger } = context;
-    const expectedChange = params.expectedChange || 'Expected UI change';
-    
-    logger?.(`[VerifyActionResult] Capturing before/after screenshots...`);
-    
-    try {
-      // Capture BEFORE screenshot (viewport only, JPEG 60% quality for cost efficiency)
-      const beforeScreenshot = await page.screenshot({ 
-        encoding: 'base64', 
-        fullPage: false,
-        type: 'jpeg',
-        quality: 60
-      });
-      const beforeDataUrl = `data:image/jpeg;base64,${beforeScreenshot}`;
-      
-      logger?.(`[VerifyActionResult] ✓ BEFORE screenshot captured (JPEG 60%)`);
-      logger?.(`[VerifyActionResult] ⏳ Waiting 1.5s for UI to settle after your previous action...`);
-      
-      // Wait for UI to settle after previous action
-      await page.waitForTimeout(1500);
-      
-      // Capture AFTER screenshot
-      const afterScreenshot = await page.screenshot({ 
-        encoding: 'base64', 
-        fullPage: false,
-        type: 'jpeg',
-        quality: 60
-      });
-      const afterDataUrl = `data:image/jpeg;base64,${afterScreenshot}`;
-      
-      logger?.(`[VerifyActionResult] ✓ AFTER screenshot captured (JPEG 60%)`);
-      logger?.(`[VerifyActionResult] 🔍 Calling vision model to compare...`);
-      
-      if (!this.llmFacade) {
-        throw new Error('LLM facade not initialized');
-      }
-      
-      // Call LLM with both screenshots
-      const request = {
-        model: 'gpt-5-mini', // Vision model
-        systemPrompt: 'You are a visual verification expert for web automation. Compare before/after screenshots to determine if an action achieved its expected outcome.',
-        userPrompt: `Expected change: ${expectedChange}
-
-Compare the [BEFORE] and [AFTER] screenshots.
-
-Did the expected change occur? Respond with JSON:
-{
-  "verified": boolean,
-  "reasoning": "What changed (or didn't change) between screenshots",
-  "changes": ["Specific UI changes observed"]
-}
-
-Be objective:
-- Look for new elements, panels, forms, modals
-- Check for page navigation, URL changes, content changes
-- Note any visual indicators of success/failure
-
-Be strict: Only return true if you clearly see the expected change.`,
-        images: [
-          { label: 'BEFORE', dataUrl: beforeDataUrl },
-          { label: 'AFTER', dataUrl: afterDataUrl }
-        ]
-      };
-      
-      const response = await this.llmFacade.llmProvider.callLLM(request);
-      
-      // Parse response
-      const jsonMatch = response.answer.match(/\{[\s\S]*\}/);
-      if (!jsonMatch) {
-        logger?.(`[VerifyActionResult] ⚠️ Could not parse verification response`, 'warn');
-        return {
-          success: false,
-          error: 'Failed to parse verification response'
-        };
-      }
-      
-      const parsed = JSON.parse(jsonMatch[0]);
-      
-      logger?.(`[VerifyActionResult] 📊 Result: ${parsed.verified ? '✅ VERIFIED' : '❌ NOT VERIFIED'}`);
-      logger?.(`[VerifyActionResult] 💭 ${parsed.reasoning}`);
-      if (parsed.changes && parsed.changes.length > 0) {
-        logger?.(`[VerifyActionResult] 📝 Changes observed:`);
-        parsed.changes.forEach((change: string) => logger?.(`   - ${change}`));
-      }
-      
-      return {
-        success: true,
-        data: parsed,
-        learning: `Verification result: ${parsed.verified ? 'Goal achieved' : 'Goal not achieved'}. ${parsed.reasoning}`
-      };
-      
-    } catch (error: any) {
-      logger?.(`[VerifyActionResult] ❌ Error: ${error.message}`, 'error');
-      return {
-        success: false,
-        error: error.message
-      };
-    }
-  }
-}
-