testchimp-runner-core 0.0.21 → 0.0.23

package/src/execution-service.ts

@@ -12,11 +12,15 @@ import {
 } from './types';
 import { RepairSuggestionResponse, RepairConfidenceResponse } from './llm-facade';
 import { getEnhancedPageInfo, PageInfo } from './utils/page-info-utils';
-import { initializeBrowser } from './utils/browser-utils';
+import { initializeBrowser, captureOptimizedScreenshot } from './utils/browser-utils';
 import { LLMFacade } from './llm-facade';
 import { AuthConfig } from './auth-config';
 import { addTestChimpComment } from './script-utils';
 import { CreditUsageService } from './credit-usage-service';
+import { DEFAULT_MODEL, VISION_MODEL } from './model-constants';
+import { LLMProvider } from './llm-provider';
+import { ProgressReporter } from './progress-reporter';
+import { BackendProxyLLMProvider } from './providers/backend-proxy-llm-provider';
 
 /**
  * Service for orchestrating Playwright script execution
@@ -24,14 +28,27 @@ import { CreditUsageService } from './credit-usage-service';
 export class ExecutionService {
   private playwrightService: PlaywrightService;
   private llmFacade: LLMFacade;
+  private llmProvider: LLMProvider;
+  private progressReporter?: ProgressReporter;
   private creditUsageService: CreditUsageService;
   private maxConcurrentExecutions: number;
   private activeExecutions: Set<Promise<any>> = new Set();
   private logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
 
-  constructor(authConfig?: AuthConfig, backendUrl?: string, maxConcurrentExecutions: number = 10) {
+  constructor(
+    authConfig?: AuthConfig,
+    backendUrl?: string,
+    maxConcurrentExecutions: number = 10,
+    llmProvider?: LLMProvider,
+    progressReporter?: ProgressReporter
+  ) {
     this.playwrightService = new PlaywrightService();
-    this.llmFacade = new LLMFacade(authConfig, backendUrl);
+    
+    // Use provided LLM provider or default to backend proxy (backward compatible)
+    this.llmProvider = llmProvider || new BackendProxyLLMProvider(authConfig, backendUrl);
+    this.llmFacade = new LLMFacade(this.llmProvider);
+    
+    this.progressReporter = progressReporter;
     this.creditUsageService = new CreditUsageService(authConfig, backendUrl);
     this.maxConcurrentExecutions = maxConcurrentExecutions;
   }
@@ -44,20 +61,13 @@ export class ExecutionService {
   }
 
   /**
-   * Log a message using the configured logger or console
+   * Log a message using the configured logger
    */
   private log(message: string, level: 'log' | 'error' | 'warn' = 'log'): void {
     if (this.logger) {
       this.logger(message, level);
-    } else {
-      if (level === 'error') {
-        console.error(message);
-      } else if (level === 'warn') {
-        console.warn(message);
-      } else {
-        console.log(message);
-      }
     }
+    // No console fallback - logs are routed to consumer
   }
 
   /**
@@ -69,9 +79,12 @@ export class ExecutionService {
 
   /**
    * Set authentication configuration for the service
+   * Note: This recreates the LLM provider with new auth config
    */
   setAuthConfig(authConfig: AuthConfig): void {
-    this.llmFacade.setAuthConfig(authConfig);
+    // Recreate LLM provider with new auth config
+    this.llmProvider = new BackendProxyLLMProvider(authConfig, undefined);
+    this.llmFacade = new LLMFacade(this.llmProvider);
     this.creditUsageService.setAuthConfig(authConfig);
   }
 
@@ -102,7 +115,7 @@ export class ExecutionService {
    */
   private async executeScriptInternal(request: ScriptExecutionRequest): Promise<ScriptExecutionResponse> {
     const startTime = Date.now();
-    const model = request.model || 'gpt-4.1-mini';
+    const model = request.model || DEFAULT_MODEL;
     
     try {
       if (request.mode === ExecutionMode.RUN_EXACTLY) {
@@ -327,7 +340,7 @@ export class ExecutionService {
           
           // Report credit usage for successful AI repair
           this.creditUsageService.reportAIRepairCredit().catch(error => {
-            console.warn(`Failed to report credit usage for AI repair:`, error);
+            this.log(`Failed to report credit usage for AI repair: ${error}`, 'warn');
           });
           
           await repairBrowser.close();
@@ -488,6 +501,7 @@ export class ExecutionService {
         let repairSuccess = false;
         const originalDescription = step.description;
         const originalCode = step.code;
+        let usedVisionMode = false;
         
         for (let attempt = 1; attempt <= maxTries; attempt++) {
           this.log(`Step ${i + 1} repair attempt ${attempt}/${maxTries}`);
@@ -501,16 +515,109 @@ export class ExecutionService {
           // Build recent repairs context for LLM
           const recentRepairsContext = this.buildRecentRepairsContext(recentRepairs);
           
-          // Ask AI for repair suggestion with failure history and recent repairs
-          const repairSuggestion = await this.llmFacade.getRepairSuggestion(
-            step.description,
-            step.code,
-            step.error || 'Unknown error',
-            pageInfo,
-            failureHistory,
-            recentRepairsContext,
-            model
-          );
+          let repairSuggestion;
+          
+          // VISION-BASED FALLBACK: After 2 regular repair attempts, consider vision diagnostics on final attempt
+          if (attempt === maxTries - 1 && repairHistory.length >= 2 && !usedVisionMode) {
+            // Ask LLM if screenshot would help for repair diagnostics
+            this.log(`  🤔 After ${repairHistory.length} failed repairs: Asking LLM if screenshot would help (last resort)...`);
+            
+            const screenshotNeed = await this.llmFacade.assessScreenshotNeed(
+              step.description,
+              step.error || 'Unknown error',
+              repairHistory.length + 1,
+              pageInfo,
+              model
+            );
+            
+            this.log(`  💭 LLM assessment: ${screenshotNeed.needsScreenshot ? 'SCREENSHOT NEEDED' : 'NO SCREENSHOT'} - ${screenshotNeed.reason}`);
+            
+            if (screenshotNeed.needsScreenshot) {
+              // Two-step supervisor pattern for vision-based repair:
+              // 1. Supervisor analyzes screenshot and provides diagnostic insights
+              // 2. Get repair suggestion with enhanced context from vision analysis
+              
+              this.log(`  📸 Taking screenshot for supervisor analysis...`);
+              
+              // Capture optimized screenshot using utility method
+              const imageDataUrl = await captureOptimizedScreenshot(
+                page,
+                { timeout: 10000 }, // Uses default quality 60
+                (msg) => this.log(msg)
+              );
+              
+              this.log(`  👔 STEP 1: Supervisor analyzing screenshot (${VISION_MODEL})...`);
+              const supervisorDiagnostics = await this.llmFacade.getVisionDiagnostics(
+                step.description,
+                pageInfo,
+                [], // No previous steps context for repair
+                step.error,
+                imageDataUrl,
+                VISION_MODEL
+              );
+              
+              // DEBUG: Log vision diagnostics
+              this.log(`  📸 Visual insights: ${supervisorDiagnostics.visualAnalysis}`);
+              this.log(`  🔍 Root cause: ${supervisorDiagnostics.rootCause}`);
+              this.log(`  💡 Recommended approach: ${supervisorDiagnostics.recommendedApproach}`);
+              if (supervisorDiagnostics.elementsFound.length > 0) {
+                this.log(`  ✅ Elements found: ${supervisorDiagnostics.elementsFound.join(', ')}`);
+              }
+              if (supervisorDiagnostics.elementsNotFound.length > 0) {
+                this.log(`  ❌ Elements not found: ${supervisorDiagnostics.elementsNotFound.join(', ')}`);
+              }
+              
+              // Get repair suggestion with vision-enhanced context
+              this.log(`  🔨 STEP 2: Getting repair suggestion with vision insights...`);
+              const visionEnhancedFailureHistory = `${failureHistory}
+
+VISION-BASED DIAGNOSTIC INSIGHTS:
+Visual Analysis: ${supervisorDiagnostics.visualAnalysis}
+Root Cause: ${supervisorDiagnostics.rootCause}
+Recommended Approach: ${supervisorDiagnostics.recommendedApproach}
+Elements Found: ${supervisorDiagnostics.elementsFound.join(', ') || 'None'}
+Elements Not Found: ${supervisorDiagnostics.elementsNotFound.join(', ') || 'None'}
+
+Use these vision insights to inform your repair strategy.`;
+              
+              repairSuggestion = await this.llmFacade.getRepairSuggestion(
+                step.description,
+                step.code,
+                step.error || 'Unknown error',
+                pageInfo,
+                visionEnhancedFailureHistory,
+                recentRepairsContext,
+                model
+              );
+              
+              usedVisionMode = true;
+            } else {
+              // Regular repair without vision
+              if (screenshotNeed.alternativeApproach) {
+                this.log(`  💡 Alternative approach: ${screenshotNeed.alternativeApproach}`);
+              }
+              repairSuggestion = await this.llmFacade.getRepairSuggestion(
+                step.description,
+                step.code,
+                step.error || 'Unknown error',
+                pageInfo,
+                failureHistory,
+                recentRepairsContext,
+                model
+              );
+            }
+          } else {
+            // Regular repair attempt (first 2 attempts or already used vision)
+            repairSuggestion = await this.llmFacade.getRepairSuggestion(
+              step.description,
+              step.code,
+              step.error || 'Unknown error',
+              pageInfo,
+              failureHistory,
+              recentRepairsContext,
+              model
+            );
+          }
 
           if (!repairSuggestion.shouldContinue) {
             this.log(`AI decided to stop repair at attempt ${attempt}: ${repairSuggestion.reason}`);
@@ -562,7 +669,7 @@ export class ExecutionService {
                 repairAction.operation === StepOperation.REMOVE ? 
                 `REMOVE: step at index ${repairAction.stepIndex}` :
                 repairAction.operation;
-              this.log(`Step ${i + 1} repair action ${commandInfo} executed successfully on attempt ${attempt}`);
+              this.log(`Step ${i + 1} repair action ${commandInfo} executed successfully on attempt ${attempt}${usedVisionMode ? ' (vision-aided)' : ''}`);
               
               // Update execution context based on the repair action
               if (repairAction.operation === StepOperation.MODIFY && repairAction.newStep) {
@@ -661,8 +768,9 @@ export class ExecutionService {
   }
 
   private async executeStepCode(code: string, page: any): Promise<void> {
-    // Set timeout for individual step execution
-    page.setDefaultTimeout(5000); // 5 seconds for individual commands
+    // Keep default timeout (5 seconds) for fast feedback on wrong selectors
+    // Navigation operations should use explicit longer timeouts in generated code
+    page.setDefaultTimeout(5000);
     
     try {
       // Clean and validate the code before execution
@@ -680,8 +788,8 @@ export class ExecutionService {
       const result = executeCode(page, expect);
       await result;
     } finally {
-      // Restore to reasonable default timeout
-      page.setDefaultTimeout(10000);
+      // Ensure timeout remains consistent
+      page.setDefaultTimeout(5000);
     }
   }
 
@@ -709,8 +817,8 @@ export class ExecutionService {
     executionContext: string, 
     contextVariables: Map<string, any>
   ): Promise<void> {
-    // Set timeout for individual step execution
-    page.setDefaultTimeout(5000); // 5 seconds for individual commands
+    // Keep default timeout (5 seconds) for fast feedback
+    page.setDefaultTimeout(5000);
     
     try {
       // Execute only the current step code, but make context variables available
@@ -779,10 +887,9 @@ export class ExecutionService {
       return {
         url: page.url(),
         title: 'Unknown',
-        elements: 'Unable to extract',
-        formFields: 'Unable to extract',
-        interactiveElements: 'Unable to extract',
-        pageStructure: 'Unable to extract'
+        ariaSnapshot: null,
+        interactiveElements: [],
+        formattedElements: 'Unable to extract'
       };
     }
   }
@@ -1004,11 +1111,14 @@ export class ExecutionService {
     ];
 
     steps.forEach((step, index) => {
-      scriptLines.push(`  // Step ${index + 1}: ${step.description}`);
-      const codeLines = step.code.split('\n');
-      codeLines.forEach(line => {
-        scriptLines.push(`  ${line}`);
-      });
+      // Only add step if it has code to execute
+      if (step.code && step.code.trim().length > 0) {
+        scriptLines.push(`  // ${step.description}`);
+        const codeLines = step.code.split('\n');
+        codeLines.forEach(line => {
+          scriptLines.push(`  ${line}`);
+        });
+      }
     });
 
     scriptLines.push('});');
@@ -1023,7 +1133,7 @@ export class ExecutionService {
    * Initialize browser with configuration (delegates to utility function)
    */
   private async initializeBrowser(playwrightConfig?: string, headless?: boolean, playwrightConfigFilePath?: string): Promise<{ browser: any; context: any; page: any }> {
-    return initializeBrowser(playwrightConfig, headless, playwrightConfigFilePath);
+    return initializeBrowser(playwrightConfig, headless, playwrightConfigFilePath, this.logger);
   }
 
   /**

package/src/index.ts

@@ -18,8 +18,45 @@ export { ExecutionService, ScenarioService, ScenarioWorker, PlaywrightMCPService
 import { FileHandler, LocalFileHandler, CIFileHandler, NoOpFileHandler } from './file-handler';
 export { FileHandler, LocalFileHandler, CIFileHandler, NoOpFileHandler };
 
+// LLM Provider interfaces
+import { LLMProvider, LLMRequest, LLMResponse } from './llm-provider';
+import { ProgressReporter, StepProgress, JobProgress, StepExecutionStatus } from './progress-reporter';
+import { BackendProxyLLMProvider } from './providers/backend-proxy-llm-provider';
+import { LocalLLMProvider } from './providers/local-llm-provider';
+
+export { LLMProvider, LLMRequest, LLMResponse };
+export { ProgressReporter, StepProgress, JobProgress, StepExecutionStatus };
+export { BackendProxyLLMProvider, LocalLLMProvider };
+
+// Orchestrator (tool-using agent)
+import type { AgentConfig } from './orchestrator';
+export {
+  OrchestratorAgent,
+  ToolRegistry,
+  Tool,
+  ToolParameter,
+  ToolExecutionContext,
+  AgentConfig,
+  AgentContext,
+  AgentDecision,
+  JourneyMemory,
+  MemoryStep,
+  OrchestratorStepResult,
+  SelfReflection,
+  ToolCall,
+  ToolResult,
+  DEFAULT_AGENT_CONFIG,
+  // Tools (information-gathering only)
+  TakeScreenshotTool,
+  RecallHistoryTool,
+  InspectPageTool,
+  CheckPageReadyTool,
+  ExtractDataTool
+} from './orchestrator';
+
 // Types
 export * from './types';
+export { PageInfo, InteractiveElement } from './utils/page-info-utils';
 
 // Authentication
 export * from './auth-config';
@@ -35,39 +72,96 @@ export class TestChimpService {
   private executionService: ExecutionService;
   public scenarioService: ScenarioService; // Make public for event listening
   private playwrightService: PlaywrightMCPService;
-  private llmFacade: LLMFacade;
+  private llmProvider: LLMProvider;
+  private progressReporter?: ProgressReporter;
   private creditUsageService: CreditUsageService;
   private fileHandler: FileHandler;
   private authConfig: AuthConfig | null;
   private backendUrl: string;
+  private logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
+  private orchestratorOptions?: { useOrchestrator?: boolean; orchestratorConfig?: Partial<AgentConfig>; debugMode?: boolean };
 
-  constructor(fileHandler?: FileHandler, authConfig?: AuthConfig, backendUrl?: string, maxWorkers?: number) {
+  constructor(
+    fileHandler?: FileHandler,
+    authConfig?: AuthConfig,
+    backendUrl?: string,
+    maxWorkers?: number,
+    llmProvider?: LLMProvider,
+    progressReporter?: ProgressReporter,
+    orchestratorOptions?: { useOrchestrator?: boolean; orchestratorConfig?: Partial<AgentConfig>; debugMode?: boolean }
+  ) {
     this.fileHandler = fileHandler || new NoOpFileHandler();
     this.authConfig = authConfig || null;
     this.backendUrl = backendUrl || 'https://featureservice.testchimp.io'; // Default to production
+    this.progressReporter = progressReporter;
+    this.orchestratorOptions = orchestratorOptions;
+    
+    // Use provided LLM provider or default to backend proxy (backward compatible)
+    this.llmProvider = llmProvider || new BackendProxyLLMProvider(authConfig, backendUrl);
+    
     this.playwrightService = new PlaywrightMCPService();
-    this.llmFacade = new LLMFacade(this.authConfig || undefined, this.backendUrl);
     this.creditUsageService = new CreditUsageService(this.authConfig || undefined, this.backendUrl);
-    this.executionService = new ExecutionService(this.authConfig || undefined, this.backendUrl, maxWorkers || 10);
-    // Set the credit usage service for the execution service
-    this.executionService.setAuthConfig(this.authConfig || {} as AuthConfig);
-    this.scenarioService = new ScenarioService(maxWorkers || 2, this.fileHandler, this.authConfig || undefined, this.backendUrl);
+    
+    // Create services with providers
+    this.executionService = new ExecutionService(
+      this.authConfig || undefined,
+      this.backendUrl,
+      maxWorkers || 10,
+      this.llmProvider,  // Pass the LLM provider
+      this.progressReporter  // Pass the progress reporter
+    );
+    
+    this.scenarioService = new ScenarioService(
+      maxWorkers || 2,
+      this.fileHandler,
+      this.llmProvider,
+      this.progressReporter,
+      this.authConfig || undefined,
+      this.backendUrl,
+      this.orchestratorOptions  // Pass orchestrator options
+    );
   }
 
   /**
    * Set authentication configuration for the service
+   * Recreates LLM provider and services with new auth config
    */
   async setAuthConfig(authConfig: AuthConfig): Promise<void> {
     this.authConfig = authConfig;
-    this.llmFacade.setAuthConfig(authConfig);
     this.creditUsageService.setAuthConfig(authConfig);
     
-    // Recreate services with new auth config to ensure all workers get the updated config
-    this.executionService = new ExecutionService(this.authConfig, this.backendUrl, 10);
-    this.scenarioService = new ScenarioService(2, this.fileHandler, this.authConfig, this.backendUrl);
+    // Recreate LLM provider with new auth config
+    this.llmProvider = new BackendProxyLLMProvider(authConfig, this.backendUrl);
     
-    // Set auth config for the execution service
-    this.executionService.setAuthConfig(authConfig);
+    // Set logger on new provider if we have one
+    if (this.logger) {
+      this.llmProvider.setLogger?.(this.logger);
+    }
+    
+    // Recreate services with new provider AND pass llmProvider and progressReporter
+    this.executionService = new ExecutionService(
+      this.authConfig,
+      this.backendUrl,
+      10,
+      this.llmProvider,  // Pass the LLM provider
+      this.progressReporter  // Pass the progress reporter
+    );
+    
+    this.scenarioService = new ScenarioService(
+      2,
+      this.fileHandler,
+      this.llmProvider,
+      this.progressReporter,
+      this.authConfig,
+      this.backendUrl,
+      this.orchestratorOptions  // Pass orchestrator options
+    );
+    
+    // Set logger on recreated services if we have one
+    if (this.logger) {
+      this.executionService.setLogger(this.logger);
+      this.scenarioService.setLogger(this.logger);
+    }
     
     // Reinitialize the services
     await this.executionService.initialize();
@@ -76,18 +170,43 @@ export class TestChimpService {
 
   /**
    * Set backend URL for the service
+   * Recreates LLM provider and services with new backend URL
    */
   setBackendUrl(backendUrl: string): void {
     this.backendUrl = backendUrl;
-    // Recreate services with new backend URL
-    this.llmFacade = new LLMFacade(this.authConfig || undefined, this.backendUrl);
+    
+    // Recreate LLM provider with new backend URL
+    this.llmProvider = new BackendProxyLLMProvider(this.authConfig || undefined, backendUrl);
+    
+    // Set logger on new provider if we have one
+    if (this.logger) {
+      this.llmProvider.setLogger?.(this.logger);
+    }
+    
+    // Recreate services with new provider
     this.creditUsageService = new CreditUsageService(this.authConfig || undefined, this.backendUrl);
-    this.executionService = new ExecutionService(this.authConfig || undefined, this.backendUrl, 10);
-    this.scenarioService = new ScenarioService(2, this.fileHandler, this.authConfig || undefined, this.backendUrl);
+    this.executionService = new ExecutionService(
+      this.authConfig || undefined,
+      this.backendUrl,
+      10,
+      this.llmProvider,  // Pass the LLM provider
+      this.progressReporter  // Pass the progress reporter
+    );
+    
+    this.scenarioService = new ScenarioService(
+      2,
+      this.fileHandler,
+      this.llmProvider,
+      this.progressReporter,
+      this.authConfig || undefined,
+      this.backendUrl,
+      this.orchestratorOptions  // Pass orchestrator options
+    );
     
-    // Set auth config for the execution service
-    if (this.authConfig) {
-      this.executionService.setAuthConfig(this.authConfig);
+    // Set logger on recreated services if we have one
+    if (this.logger) {
+      this.executionService.setLogger(this.logger);
+      this.scenarioService.setLogger(this.logger);
     }
   }
 
@@ -95,7 +214,31 @@ export class TestChimpService {
    * Set logger callback for capturing execution logs
    */
   setLogger(logger: (message: string, level?: 'log' | 'error' | 'warn') => void): void {
+    this.logger = logger;
     this.executionService.setLogger(logger);
+    this.scenarioService.setLogger(logger);
+    this.llmProvider.setLogger?.(logger);
+    this.creditUsageService.setLogger(logger);
+  }
+  
+  /**
+   * Set output channel for worker logs (VS Code OutputChannel)
+   * This enables orchestrator thinking logs to appear in output console
+   */
+  setOutputChannel(outputChannel: any): void {
+    if (typeof this.scenarioService?.setOutputChannel === 'function') {
+      this.scenarioService.setOutputChannel(outputChannel);
+    }
+  }
+
+  /**
+   * Log a message using the configured logger
+   */
+  private log(message: string, level: 'log' | 'error' | 'warn' = 'log'): void {
+    if (this.logger) {
+      this.logger(message, level);
+    }
+    // No console fallback - logs are routed to consumer
   }
 
   /**
@@ -127,7 +270,7 @@ export class TestChimpService {
       try {
         const resolvedPath = this.fileHandler.resolvePath(request.scriptFilePath);
         request.script = await this.fileHandler.readTestFile(resolvedPath);
-        console.log(`Read script content from file: ${resolvedPath}`);
+        this.log(`Read script content from file: ${resolvedPath}`);
       } catch (error) {
         throw new Error(`Failed to read script file: ${error}`);
       }
@@ -138,24 +281,24 @@ export class TestChimpService {
       try {
         const resolvedPath = this.fileHandler.resolvePath(request.playwrightConfigFilePath);
         request.playwrightConfig = await this.fileHandler.readTestFile(resolvedPath);
-        console.log(`Read Playwright config content from file: ${resolvedPath}`);
+        this.log(`Read Playwright config content from file: ${resolvedPath}`);
       } catch (error) {
-        console.warn(`Failed to read Playwright config file: ${error}. Using default configuration.`);
+        this.log(`Failed to read Playwright config file: ${error}. Using default configuration.`, 'warn');
         // Don't throw error, just use default config
       }
     }
 
     // Log content status
     if (request.script) {
-      console.log(`Using provided script content (${request.script.length} characters)`);
+      this.log(`Using provided script content (${request.script.length} characters)`);
     } else {
       throw new Error('Script content is required. Provide either script or scriptFilePath.');
     }
 
     if (request.playwrightConfig) {
-      console.log(`Using provided Playwright config (${request.playwrightConfig.length} characters)`);
+      this.log(`Using provided Playwright config (${request.playwrightConfig.length} characters)`);
     } else {
-      console.log(`Using default Playwright configuration`);
+      this.log(`Using default Playwright configuration`);
     }
 
     const result = await this.executionService.executeScript(request);