testchimp-runner-core 0.0.21 → 0.0.23

package/src/orchestrator/tool-registry.ts

@@ -0,0 +1,182 @@
+/**
+ * Tool Registry - Dynamic tool registration and prompt generation
+ * Tools can be added at runtime and their descriptions are automatically included in agent prompts
+ */
+
+import { ToolCall, ToolResult } from './types';
+
+/**
+ * Tool parameter definition
+ */
+export interface ToolParameter {
+  name: string;
+  type: 'string' | 'number' | 'boolean' | 'object';
+  description: string;
+  required: boolean;
+  default?: any;
+}
+
+/**
+ * Tool definition
+ */
+export interface Tool {
+  name: string;
+  description: string;
+  parameters: ToolParameter[];
+  
+  /**
+   * Execute the tool
+   * @param params Tool parameters
+   * @param context Execution context (page, memory, etc.)
+   */
+  execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult>;
+}
+
+/**
+ * Context provided to tool execution
+ */
+export interface ToolExecutionContext {
+  page: any;  // Playwright Page
+  memory: any;  // JourneyMemory
+  stepNumber: number;
+  logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
+}
+
+/**
+ * Tool Registry - manages available tools and generates prompts
+ */
+export class ToolRegistry {
+  private tools: Map<string, Tool> = new Map();
+  
+  /**
+   * Register a tool
+   */
+  register(tool: Tool): void {
+    this.tools.set(tool.name, tool);
+  }
+  
+  /**
+   * Unregister a tool
+   */
+  unregister(toolName: string): void {
+    this.tools.delete(toolName);
+  }
+  
+  /**
+   * Get a tool by name
+   */
+  get(toolName: string): Tool | undefined {
+    return this.tools.get(toolName);
+  }
+  
+  /**
+   * Get all registered tools
+   */
+  getAll(): Tool[] {
+    return Array.from(this.tools.values());
+  }
+  
+  /**
+   * Execute a tool
+   */
+  async execute(toolCall: ToolCall, context: ToolExecutionContext): Promise<ToolResult> {
+    const tool = this.tools.get(toolCall.name);
+    
+    if (!tool) {
+      return {
+        success: false,
+        error: `Tool '${toolCall.name}' not found`
+      };
+    }
+    
+    // Validate required parameters
+    const missingParams = tool.parameters
+      .filter(p => p.required && !(p.name in toolCall.params))
+      .map(p => p.name);
+    
+    if (missingParams.length > 0) {
+      return {
+        success: false,
+        error: `Missing required parameters: ${missingParams.join(', ')}`
+      };
+    }
+    
+    // Apply defaults for missing optional parameters
+    const params = { ...toolCall.params };
+    for (const param of tool.parameters) {
+      if (!param.required && !(param.name in params) && param.default !== undefined) {
+        params[param.name] = param.default;
+      }
+    }
+    
+    try {
+      return await tool.execute(params, context);
+    } catch (error: any) {
+      return {
+        success: false,
+        error: `Tool execution failed: ${error.message}`
+      };
+    }
+  }
+  
+  /**
+   * Generate tool descriptions for agent prompt
+   * Returns formatted text describing all available tools
+   */
+  generateToolDescriptions(): string {
+    if (this.tools.size === 0) {
+      return 'No tools available.';
+    }
+    
+    const descriptions: string[] = [];
+    
+    descriptions.push('AVAILABLE TOOLS:');
+    descriptions.push('');
+    
+    for (const tool of this.tools.values()) {
+      descriptions.push(`${tool.name}:`);
+      descriptions.push(`  Description: ${tool.description}`);
+      
+      if (tool.parameters.length > 0) {
+        descriptions.push(`  Parameters:`);
+        for (const param of tool.parameters) {
+          const required = param.required ? '(required)' : '(optional)';
+          const defaultVal = param.default !== undefined ? ` [default: ${JSON.stringify(param.default)}]` : '';
+          descriptions.push(`    - ${param.name} (${param.type}) ${required}: ${param.description}${defaultVal}`);
+        }
+      } else {
+        descriptions.push(`  Parameters: none`);
+      }
+      
+      descriptions.push('');
+    }
+    
+    descriptions.push('To use a tool, include it in your "toolCalls" array with the tool name and parameters.');
+    descriptions.push('');
+    
+    return descriptions.join('\n');
+  }
+  
+  /**
+   * Generate JSON schema for tool calls (for structured output)
+   */
+  generateToolCallSchema(): any {
+    return {
+      type: 'array',
+      items: {
+        type: 'object',
+        properties: {
+          name: {
+            type: 'string',
+            enum: Array.from(this.tools.keys())
+          },
+          params: {
+            type: 'object'
+          }
+        },
+        required: ['name', 'params']
+      }
+    };
+  }
+}
+

package/src/orchestrator/tools/check-page-ready.ts

@@ -0,0 +1,75 @@
+/**
+ * Check Page Ready Tool
+ * Verify that page has finished loading and is interactive
+ */
+
+import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
+import { ToolResult } from '../types';
+
+export class CheckPageReadyTool implements Tool {
+  name = 'check_page_ready';
+  description = 'Verify that the page has finished loading and is ready for interaction. Checks load state, no pending network requests, and DOM stability.';
+  
+  parameters: ToolParameter[] = [
+    {
+      name: 'timeout',
+      type: 'number',
+      description: 'Maximum time to wait for page to be ready (milliseconds)',
+      required: false,
+      default: 5000
+    }
+  ];
+  
+  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
+    const { page, logger } = context;
+    const timeout = params.timeout || 5000;
+    
+    try {
+      logger?.(`[CheckPageReady] Verifying page is ready (timeout: ${timeout}ms)`, 'log');
+      
+      // Wait for load state
+      await page.waitForLoadState('domcontentloaded', { timeout });
+      
+      // Check if page is interactive (use string to avoid TypeScript DOM type issues)
+      const isInteractive = await page.evaluate(() => {
+        const doc = (globalThis as any).document;
+        return doc.readyState === 'complete' || doc.readyState === 'interactive';
+      });
+      
+      // Get page state info
+      const url = page.url();
+      const title = await page.title();
+      
+      if (isInteractive) {
+        logger?.(`[CheckPageReady] ✓ Page is ready: ${title} (${url})`, 'log');
+        return {
+          success: true,
+          data: {
+            ready: true,
+            url,
+            title,
+            readyState: 'complete'
+          }
+        };
+      } else {
+        logger?.(`[CheckPageReady] ⚠ Page not fully ready yet`, 'warn');
+        return {
+          success: true,
+          data: {
+            ready: false,
+            url,
+            title,
+            readyState: 'loading'
+          }
+        };
+      }
+    } catch (error: any) {
+      logger?.(`[CheckPageReady] ✗ Failed: ${error.message}`, 'error');
+      return {
+        success: false,
+        error: `Page ready check failed: ${error.message}`
+      };
+    }
+  }
+}
+

package/src/orchestrator/tools/extract-data.ts

@@ -0,0 +1,92 @@
+/**
+ * Extract Data Tool
+ * Save data from page for use in later steps
+ */
+
+import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
+import { ToolResult } from '../types';
+import { JourneyMemory } from '../types';
+
+export class ExtractDataTool implements Tool {
+  name = 'extract_data';
+  description = 'Extract and save data from the page for use in later steps. Useful for capturing IDs, usernames, confirmation codes, etc.';
+  
+  parameters: ToolParameter[] = [
+    {
+      name: 'selector',
+      type: 'string',
+      description: 'CSS selector or text content to extract',
+      required: true
+    },
+    {
+      name: 'dataName',
+      type: 'string',
+      description: 'Name to save the data under (e.g., "userId", "confirmationCode")',
+      required: true
+    },
+    {
+      name: 'attribute',
+      type: 'string',
+      description: 'Optional: Extract specific attribute instead of text content (e.g., "href", "value")',
+      required: false
+    }
+  ];
+  
+  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
+    const { page, memory, logger } = context;
+    const journeyMemory = memory as JourneyMemory;
+    const { selector, dataName, attribute } = params;
+    
+    try {
+      logger?.(`[ExtractData] Extracting "${dataName}" from selector: ${selector}`, 'log');
+      
+      // Try to find element
+      const element = await page.locator(selector).first();
+      const count = await page.locator(selector).count();
+      
+      if (count === 0) {
+        logger?.(`[ExtractData] ✗ Element not found: ${selector}`, 'error');
+        return {
+          success: false,
+          error: `Element not found: ${selector}`
+        };
+      }
+      
+      // Extract data
+      let extractedValue: string;
+      if (attribute) {
+        extractedValue = await element.getAttribute(attribute) || '';
+      } else {
+        extractedValue = await element.textContent() || '';
+      }
+      
+      extractedValue = extractedValue.trim();
+      
+      if (!extractedValue) {
+        logger?.(`[ExtractData] ⚠ Extracted empty value from ${selector}`, 'warn');
+      }
+      
+      // Save to memory
+      journeyMemory.extractedData[dataName] = extractedValue;
+      
+      logger?.(`[ExtractData] ✓ Saved "${dataName}" = "${extractedValue.substring(0, 50)}${extractedValue.length > 50 ? '...' : ''}"`, 'log');
+      
+      return {
+        success: true,
+        data: {
+          dataName,
+          value: extractedValue,
+          selector,
+          attribute
+        }
+      };
+    } catch (error: any) {
+      logger?.(`[ExtractData] ✗ Failed: ${error.message}`, 'error');
+      return {
+        success: false,
+        error: `Data extraction failed: ${error.message}`
+      };
+    }
+  }
+}
+

package/src/orchestrator/tools/index.ts

@@ -0,0 +1,11 @@
+/**
+ * Tool exports - 5 information-gathering tools
+ * Note: State changes (navigation, clicks, fills) are done via Playwright commands, not tools
+ */
+
+export { TakeScreenshotTool } from './take-screenshot';
+export { RecallHistoryTool } from './recall-history';
+export { InspectPageTool } from './inspect-page';
+export { CheckPageReadyTool } from './check-page-ready';
+export { ExtractDataTool } from './extract-data';
+

package/src/orchestrator/tools/inspect-page.ts

@@ -0,0 +1,42 @@
+/**
+ * Inspect Page Tool
+ * Get current DOM snapshot (might be redundant since DOM is always-provided, but keeps extensibility)
+ */
+
+import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
+import { ToolResult } from '../types';
+import { getEnhancedPageInfo } from '../../utils/page-info-utils';
+
+export class InspectPageTool implements Tool {
+  name = 'inspect_page';
+  description = 'Get current page DOM snapshot. Note: Current page info is already provided in context, so this tool is mainly for forcing a fresh snapshot if needed.';
+  
+  parameters: ToolParameter[] = [];
+  
+  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
+    const { page, logger } = context;
+    
+    try {
+      logger?.(`[InspectPage] Getting fresh DOM snapshot`, 'log');
+      
+      const pageInfo = await getEnhancedPageInfo(page);
+      
+      logger?.(`[InspectPage] ✓ DOM snapshot retrieved`, 'log');
+      
+      return {
+        success: true,
+        data: {
+          pageInfo,
+          url: page.url()
+        }
+      };
+    } catch (error: any) {
+      logger?.(`[InspectPage] ✗ Failed: ${error.message}`, 'error');
+      return {
+        success: false,
+        error: `Page inspection failed: ${error.message}`
+      };
+    }
+  }
+}
+

package/src/orchestrator/tools/recall-history.ts

@@ -0,0 +1,72 @@
+/**
+ * Recall History Tool
+ * Access deeper history beyond the recent 6-7 steps always provided
+ */
+
+import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
+import { ToolResult } from '../types';
+import { JourneyMemory } from '../types';
+
+export class RecallHistoryTool implements Tool {
+  name = 'recall_history';
+  description = 'Access journey history beyond the recent 6-7 steps always provided in context. Use when you need to remember what happened earlier in the journey.';
+  
+  parameters: ToolParameter[] = [
+    {
+      name: 'maxSteps',
+      type: 'number',
+      description: 'Maximum number of historical steps to retrieve (from most recent backwards)',
+      required: false,
+      default: 10
+    },
+    {
+      name: 'query',
+      type: 'string',
+      description: 'Optional: Filter for specific actions (e.g., "login", "form fill")',
+      required: false
+    }
+  ];
+  
+  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
+    const { memory, logger } = context;
+    const journeyMemory = memory as JourneyMemory;
+    const maxSteps = params.maxSteps || 10;
+    const query = params.query?.toLowerCase();
+    
+    try {
+      logger?.(`[RecallHistory] Retrieving up to ${maxSteps} historical steps${query ? ` matching "${query}"` : ''}`, 'log');
+      
+      let steps = [...journeyMemory.history];
+      
+      // Filter by query if provided
+      if (query) {
+        steps = steps.filter(step => 
+          step.action.toLowerCase().includes(query) ||
+          step.code.toLowerCase().includes(query) ||
+          step.observation.toLowerCase().includes(query)
+        );
+      }
+      
+      // Take most recent N steps
+      const recentSteps = steps.slice(-maxSteps);
+      
+      logger?.(`[RecallHistory] ✓ Found ${recentSteps.length} historical steps`, 'log');
+      
+      return {
+        success: true,
+        data: {
+          steps: recentSteps,
+          totalHistorySize: journeyMemory.history.length,
+          filtered: !!query
+        }
+      };
+    } catch (error: any) {
+      logger?.(`[RecallHistory] ✗ Failed: ${error.message}`, 'error');
+      return {
+        success: false,
+        error: `History recall failed: ${error.message}`
+      };
+    }
+  }
+}
+

package/src/orchestrator/tools/take-screenshot.ts

@@ -0,0 +1,128 @@
+/**
+ * Take Screenshot Tool
+ * Captures current page state visually and analyzes it with DOM snapshot
+ */
+
+import { Tool, ToolParameter, ToolExecutionContext } from '../tool-registry';
+import { ToolResult } from '../types';
+import { getEnhancedPageInfo } from '../../utils/page-info-utils';
+import { VISION_MODEL } from '../../model-constants';
+
+export class TakeScreenshotTool implements Tool {
+  name = 'take_screenshot';
+  description = 'Capture a screenshot and analyze it with DOM snapshot to get actionable selector recommendations. Use when you need to see the actual page visually to find the right elements. Returns text-based analysis with selector suggestions.';
+  
+  // LLM facade will be injected
+  private llmFacade?: any;
+  
+  setLLMFacade(llmFacade: any): void {
+    this.llmFacade = llmFacade;
+  }
+  
+  parameters: ToolParameter[] = [
+    {
+      name: 'isFullPage',
+      type: 'boolean',
+      description: 'If true, captures entire page (scrolling). If false, captures only viewport. DEFAULT: true (recommended for finding elements that may be below fold)',
+      required: false,
+      default: true
+    },
+    {
+      name: 'purpose',
+      type: 'string',
+      description: 'What you are trying to find or understand from the screenshot',
+      required: false
+    }
+  ];
+  
+  async execute(params: Record<string, any>, context: ToolExecutionContext): Promise<ToolResult> {
+    const { page, logger } = context;
+    const isFullPage = params.isFullPage !== undefined ? params.isFullPage : true;  // Default: true (capture full page)
+    const purpose = params.purpose || 'Analyze page structure and elements';
+    
+    try {
+      logger?.(`[TakeScreenshot] Capturing ${isFullPage ? 'full page' : 'viewport'} screenshot for: ${purpose}`, 'log');
+      
+      // Capture screenshot (JPEG 60 quality for smaller size)
+      const screenshotBuffer = await page.screenshot({
+        fullPage: isFullPage,
+        type: 'jpeg',
+        quality: 60
+      });
+      
+      // Convert to data URL
+      const base64 = screenshotBuffer.toString('base64');
+      const dataUrl = `data:image/jpeg;base64,${base64}`;
+      
+      logger?.(`[TakeScreenshot] ✓ Screenshot captured (${Math.round(base64.length / 1024)}KB), analyzing with DOM...`, 'log');
+      
+      // Get DOM snapshot for correlation (already has ARIA tree + interactive elements with bboxes)
+      const pageInfo = await getEnhancedPageInfo(page);
+      
+      // Analyze screenshot with structured DOM via LLM
+      let analysis = 'Screenshot captured. Use DOM snapshot in context to find selectors.';
+      
+      if (this.llmFacade) {
+        try {
+          logger?.(`[TakeScreenshot] Calling LLM for vision analysis...`, 'log');
+          
+          const analysisPrompt = `Analyze screenshot WITH DOM structure to identify correct Playwright selectors.
+
+PURPOSE: ${purpose}
+
+INTERACTIVE ELEMENTS (with positions and suggested selectors):
+${pageInfo.formattedElements}
+
+ARIA TREE (hierarchical structure):
+${JSON.stringify(pageInfo.ariaSnapshot, null, 2).substring(0, 2000)}
+
+TASK:
+1. Look at screenshot - identify the visual elements you need for: ${purpose}
+2. Match visual position with bounding boxes above
+3. Recommend SEMANTIC SELECTORS FIRST: getByRole, getByLabel, getByPlaceholder, getByText
+4. AVOID auto-generated IDs with unicode (e.g., #«r3»-form-item)
+
+Output format:
+"For [visual element description]:
+ Try: [semantic selector from list - prefer getByRole/getByLabel]
+ Or: [alternative selector]"
+
+Be concise. Only 2-3 recommendations. Prioritize user-friendly semantic selectors.`;
+
+
+          const llmResponse = await this.llmFacade.llmProvider.callLLM({
+            model: VISION_MODEL,
+            systemPrompt: 'You are a vision analysis expert for web automation. Analyze screenshots with DOM snapshots to recommend working Playwright selectors. ALWAYS prioritize semantic selectors (getByRole, getByLabel, getByText) over CSS selectors with auto-generated IDs.',
+            userPrompt: analysisPrompt,
+            imageUrl: dataUrl
+          });
+          
+          analysis = llmResponse.answer || analysis;
+          logger?.(`[TakeScreenshot] ✓ Vision analysis complete`, 'log');
+          
+        } catch (error: any) {
+          logger?.(`[TakeScreenshot] ⚠ Vision analysis failed, returning raw screenshot: ${error.message}`, 'warn');
+        }
+      }
+      
+      return {
+        success: true,
+        data: {
+          screenshot: dataUrl,
+          isFullPage,
+          size: base64.length,
+          interactiveElementCount: pageInfo.interactiveElements.length
+        },
+        learning: analysis  // Text-based analysis with selector recommendations
+      };
+    } catch (error: any) {
+      logger?.(`[TakeScreenshot] ✗ Failed: ${error.message}`, 'error');
+      return {
+        success: false,
+        error: `Screenshot capture failed: ${error.message}`
+      };
+    }
+  }
+}
+
+