testchimp-runner-core 0.0.33 → 0.0.35

package/src/index.ts

@@ -45,6 +45,7 @@ export {
   SelfReflection,
   ToolCall,
   ToolResult,
+  ExplorationMode,
   DEFAULT_AGENT_CONFIG,
   // Tools (information-gathering only)
   TakeScreenshotTool,
@@ -58,6 +59,9 @@ export {
 export * from './types';
 export { PageInfo, InteractiveElement } from './utils/page-info-utils';
 
+// Model constants
+export { DEFAULT_MODEL, DEFAULT_SIMPLER_MODEL, VISION_MODEL } from './model-constants';
+
 // Authentication
 export * from './auth-config';
 
@@ -359,6 +363,12 @@ export class TestChimpService {
     return this.executeScript(repairRequest);
   }
 
+  // Exploration mode execution
+  async executeExploration(page: any, explorationConfig: any, jobId: string): Promise<any> {
+    // Delegate to scenario service's orchestrator
+    return this.scenarioService.executeExploration(page, explorationConfig, jobId);
+  }
+
   // Credit usage reporting methods
   async reportScriptGenerationCredit(jobId?: string): Promise<void> {
     return this.creditUsageService.reportScriptGenerationCredit(jobId);

package/src/llm-facade.ts

@@ -1,7 +1,7 @@
 import { PROMPTS } from './prompts';
 import { PageInfo } from './utils/page-info-utils';
 import { StepOperation } from './types';
-import { DEFAULT_MODEL, VISION_MODEL } from './model-constants';
+import { DEFAULT_MODEL, DEFAULT_SIMPLER_MODEL, VISION_MODEL } from './model-constants';
 import { LLMProvider, LLMRequest, LLMResponse } from './llm-provider';
 
 // LLM Response interfaces
@@ -140,7 +140,7 @@ export class LLMFacade {
   /**
    * Generate a test name from scenario description
    */
-  async generateTestName(scenario: string, model: string = DEFAULT_MODEL): Promise<string> {
+  async generateTestName(scenario: string, model: string = DEFAULT_SIMPLER_MODEL): Promise<string> {
     this.log('Generating test name with LLM...');
     
     const request: LLMRequest = {
@@ -163,7 +163,7 @@ export class LLMFacade {
   /**
    * Generate hashtags for semantic grouping
    */
-  async generateHashtags(scenario: string, model: string = 'gpt-4o-mini'): Promise<string[]> {
+  async generateHashtags(scenario: string, model: string = DEFAULT_SIMPLER_MODEL): Promise<string[]> {
     this.log('Generating hashtags with LLM...');
     
     const request: LLMRequest = {
@@ -280,7 +280,7 @@ Respond ONLY with valid JSON:
     errorMessage: string,
     attemptCount: number,
     pageInfo: any,
-    model: string = DEFAULT_MODEL
+    model: string = DEFAULT_SIMPLER_MODEL
   ): Promise<ScreenshotNeedResponse> {
     this.log('Assessing screenshot need with LLM...');
     
@@ -479,7 +479,7 @@ Respond ONLY with valid JSON:
   /**
    * Break down scenario into steps
    */
-  async breakdownScenario(scenario: string, model: string = DEFAULT_MODEL): Promise<ScenarioStep[]> {
+  async breakdownScenario(scenario: string, model: string = DEFAULT_SIMPLER_MODEL): Promise<ScenarioStep[]> {
     this.log('Breaking down scenario with LLM...');
     this.log(`📝 INPUT SCENARIO: ${scenario}`);
     
@@ -571,7 +571,7 @@ Respond ONLY with valid JSON:
   /**
    * Parse script into steps for AI repair
    */
-  async parseScriptIntoSteps(script: string, model: string = 'gpt-4o-mini'): Promise<Array<{ description: string; code: string; success?: boolean; error?: string }>> {
+  async parseScriptIntoSteps(script: string, model: string = DEFAULT_SIMPLER_MODEL): Promise<Array<{ description: string; code: string; success?: boolean; error?: string }>> {
     this.log('Parsing script into steps with LLM...');
     
     const request: LLMRequest = {
@@ -657,7 +657,7 @@ Respond ONLY with valid JSON:
   async assessRepairConfidence(
     originalScript: string,
     updatedScript: string,
-    model: string = DEFAULT_MODEL
+    model: string = DEFAULT_SIMPLER_MODEL
   ): Promise<RepairConfidenceResponse> {
     const request: LLMRequest = {
       model,
@@ -676,7 +676,7 @@ Respond ONLY with valid JSON:
     originalScript: string,
     updatedScript: string,
     newRepairAdvice: string,
-    model: string = 'gpt-4o-mini'
+    model: string = DEFAULT_SIMPLER_MODEL
   ): Promise<string> {
     const request: LLMRequest = {
       model,

package/src/llm-provider.ts

@@ -3,6 +3,15 @@
  * Abstraction for LLM calls - allows different implementations (backend proxy, local, etc.)
  */
 
+/**
+ * Labeled image for multi-image LLM requests
+ * Used for before/after comparisons or multi-screenshot analysis
+ */
+export interface LabeledImage {
+  label: string;      // e.g., "Before", "After", "Screenshot 1"
+  dataUrl: string;    // Base64 data URL (data:image/png;base64,...)
+}
+
 /**
  * LLM request - camelCase for TypeScript
  */
@@ -10,7 +19,8 @@ export interface LLMRequest {
   model?: string;
   systemPrompt?: string;
   userPrompt?: string;
-  imageUrl?: string; // For vision models (data URL)
+  imageUrl?: string;           // For vision models (data URL) - backward compatible, single image
+  images?: LabeledImage[];     // For multi-image requests (e.g., before/after comparison)
 }
 
 /**

package/src/model-constants.ts

@@ -5,14 +5,26 @@
  */
 
 /**
- * Default model for general LLM tasks:
- * - Scenario breakdown
- * - Command generation
- * - Goal completion checks
- * - Screenshot need assessment
+ * Default model for complex reasoning tasks:
+ * - Command generation (Playwright selectors)
+ * - Goal completion checks (critical decisions)
+ * - Repair suggestions (complex logic)
+ * - Agent orchestration (decision-making)
  */
 export const DEFAULT_MODEL = 'gpt-5-mini';
 
+/**
+ * Simpler model for structured/guided tasks:
+ * - Scenario breakdown (structured splitting)
+ * - Screenshot need assessment (yes/no decision)
+ * - Repair confidence assessment (comparison task)
+ * - Test name generation (text formatting)
+ * - Hashtag generation (semantic tags)
+ * - Script parsing (extract steps)
+ * - Final script merging (text concatenation)
+ */
+export const DEFAULT_SIMPLER_MODEL = 'gpt-4o-mini';
+
 /**
  * Vision-capable model for visual diagnostics:
  * - Screenshot analysis

package/src/orchestrator/decision-parser.ts

@@ -0,0 +1,139 @@
+/**
+ * Decision Parser Utility
+ * Parses and validates agent decisions from LLM responses
+ */
+
+import { AgentDecision } from './types';
+import { isSomVerification, isSomCommand } from './som-types';
+
+export class DecisionParser {
+  private logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
+
+  constructor(logger?: (message: string, level?: 'log' | 'error' | 'warn') => void) {
+    this.logger = logger;
+  }
+
+  /**
+   * Parse agent decision from LLM response
+   */
+  parse(response: string): AgentDecision {
+    try {
+      // Extract JSON from response
+      const jsonMatch = response.match(/\{[\s\S]*\}/);
+      if (!jsonMatch) {
+        this.logger?.(`[DecisionParser] ✗ No JSON found in LLM response`, 'error');
+        this.logger?.(`[DecisionParser] 📄 FULL LLM RESPONSE:\n${response}`, 'error');
+        throw new Error('No JSON found in response');
+      }
+      
+      const parsed = JSON.parse(jsonMatch[0]);
+      
+      // Validate required fields
+      // Accept either "reasoning" or "statusReasoning" (LLMs sometimes only provide one)
+      if (!parsed.status || (!parsed.reasoning && !parsed.statusReasoning)) {
+        this.logger?.(`[DecisionParser] ✗ Missing required fields in parsed JSON`, 'error');
+        this.logger?.(`[DecisionParser] 📄 FULL LLM RESPONSE:\n${response}`, 'error');
+        this.logger?.(`[DecisionParser] 📄 PARSED JSON:\n${JSON.stringify(parsed, null, 2)}`, 'error');
+        this.logger?.(`[DecisionParser] ❌ Has status: ${!!parsed.status}, Has reasoning: ${!!parsed.reasoning}, Has statusReasoning: ${!!parsed.statusReasoning}`, 'error');
+        throw new Error('Missing required fields: status and (reasoning or statusReasoning)');
+      }
+      
+      // Normalize: if reasoning is missing but statusReasoning exists, use statusReasoning as reasoning
+      if (!parsed.reasoning && parsed.statusReasoning) {
+        parsed.reasoning = parsed.statusReasoning;
+      }
+      
+      return parsed as AgentDecision;
+      
+    } catch (error: any) {
+      this.logger?.(`[DecisionParser] ✗ Failed to parse agent decision: ${error.message}`, 'error');
+      
+      // Only log full response if not already logged above
+      if (!error.message.includes('Missing required fields') && !error.message.includes('No JSON found')) {
+        this.logger?.(`[DecisionParser] 📄 FULL LLM RESPONSE:\n${response}`, 'error');
+      }
+      
+      // Return fallback
+      return {
+        status: 'stuck',
+        statusReasoning: 'Failed to parse agent response',
+        reasoning: `Parse error: ${error.message}`
+      };
+    }
+  }
+
+  /**
+   * Log agent decision for debugging
+   */
+  log(decision: AgentDecision, iteration: number): void {
+    this.logger?.(`[Orchestrator] 💭 REASONING: ${decision.reasoning}`);
+    
+    if (decision.selfReflection) {
+      this.logger?.(`[Orchestrator] 🧠 SELF-REFLECTION:`);
+      this.logger?.(`[Orchestrator]    Next: ${decision.selfReflection.guidanceForNext}`);
+      if (decision.selfReflection.detectingLoop) {
+        this.logger?.(`[Orchestrator]    🔄 LOOP DETECTED: ${decision.selfReflection.loopReasoning}`, 'warn');
+      }
+    }
+    
+    if (decision.toolCalls && decision.toolCalls.length > 0) {
+      this.logger?.(`[Orchestrator] 🔧 TOOLS: ${decision.toolCalls.map(t => t.name).join(', ')}`);
+      if (decision.toolReasoning) {
+        this.logger?.(`[Orchestrator] 📋 Why: ${decision.toolReasoning}`);
+      }
+    }
+    
+    if (decision.blockerDetected) {
+      this.logger?.(`[Orchestrator] 🚧 BLOCKER: ${decision.blockerDetected.description}`, 'warn');
+      this.logger?.(`[Orchestrator] 🧹 Clearing with ${decision.blockerDetected.clearingCommands.length} command(s)`);
+    }
+    
+    if (decision.stepReEvaluation?.detected) {
+      this.logger?.(`[Orchestrator] 🔍 STEP RE-EVALUATION: ${decision.stepReEvaluation.issue}`, 'warn');
+      this.logger?.(`[Orchestrator] 📝 Explanation: ${decision.stepReEvaluation.explanation}`);
+    }
+    
+    if (decision.commands && decision.commands.length > 0) {
+      this.logger?.(`[Orchestrator] 📝 COMMANDS (${decision.commands.length}):`);
+      decision.commands.slice(0, 3).forEach((cmd: any, i) => {
+        // Handle string commands (standard mode), SomCommand, and SomVerification objects
+        if (typeof cmd === 'string') {
+          this.logger?.(`[Orchestrator]    ${i + 1}. ${cmd.substring(0, 80)}...`);
+        } else if (isSomVerification(cmd)) {
+          const ref = cmd.elementRef || cmd.selector || 'unknown';
+          this.logger?.(`[Orchestrator]    ${i + 1}. [VERIFY] ${cmd.verificationType} on ${ref}: ${cmd.expected}`);
+        } else if (isSomCommand(cmd)) {
+          this.logger?.(`[Orchestrator]    ${i + 1}. [ACTION] ${cmd.action} on ${cmd.elementRef || 'coord'}`);
+        } else {
+          this.logger?.(`[Orchestrator]    ${i + 1}. ${JSON.stringify(cmd).substring(0, 80)}...`);
+        }
+      });
+      if (decision.commands.length > 3) {
+        this.logger?.(`[Orchestrator]    ... and ${decision.commands.length - 3} more`);
+      }
+      if (decision.commandReasoning) {
+        this.logger?.(`[Orchestrator] 💡 Why: ${decision.commandReasoning}`);
+      }
+    }
+    
+    // Log meta-learning suggestions with clear demarcations
+    if (decision.debugInfo?.suggestedPromptUpdates) {
+      this.logger?.(``, 'warn');
+      this.logger?.(`═══════════════════════════════════════════════════════════════`, 'warn');
+      this.logger?.(`[Orchestrator] 🧠 META-LEARNING SUGGESTION FROM AGENT`, 'warn');
+      this.logger?.(`═══════════════════════════════════════════════════════════════`, 'warn');
+      this.logger?.(``, 'warn');
+      this.logger?.(`SUGGESTED PROMPT UPDATE:`, 'warn');
+      this.logger?.(`  ${decision.debugInfo.suggestedPromptUpdates}`, 'warn');
+      this.logger?.(``, 'warn');
+      if (decision.debugInfo.reasoning) {
+        this.logger?.(`REASONING:`, 'warn');
+        this.logger?.(`  ${decision.debugInfo.reasoning}`, 'warn');
+        this.logger?.(``, 'warn');
+      }
+      this.logger?.(`═══════════════════════════════════════════════════════════════`, 'warn');
+      this.logger?.(``, 'warn');
+    }
+  }
+}
+

package/src/orchestrator/index.ts

@@ -5,16 +5,22 @@
 // Core orchestrator
 export { OrchestratorAgent } from './orchestrator-agent';
 
+// SoM (Set-of-Marks) handler
+export { PageSoMHandler } from './page-som-handler';
+
 // Tool system
 export { ToolRegistry, Tool, ToolParameter, ToolExecutionContext } from './tool-registry';
 
-// 5 information-gathering tools (state changes via Playwright commands)
+// 8 information-gathering tools (state changes via Playwright commands)
 export {
   TakeScreenshotTool,
+  ViewPreviousScreenshotTool,
+  RefreshSomMarkersTool,
   RecallHistoryTool,
   InspectPageTool,
   CheckPageReadyTool,
-  ExtractDataTool
+  ExtractDataTool,
+  VerifyActionResultTool
 } from './tools';
 
 // Types
@@ -28,6 +34,25 @@ export {
   SelfReflection,
   ToolCall,
   ToolResult,
+  ExplorationMode,
   DEFAULT_AGENT_CONFIG
 } from './types';
 
+// SoM Types
+export {
+  SomCommand,
+  SomElement,
+  SemanticCommandResult,
+  CommandRunStatus,
+  CommandAttempt,
+  DomMutation,
+  InteractionAction,
+  Coordinate,
+  TypedSelector,
+  SomVerification,
+  VerificationType,
+  SomCommandOrVerification,
+  isSomVerification,
+  isSomCommand
+} from './som-types';
+