testchimp-runner-core 0.0.15 → 0.0.17

package/src/execution-service.ts

@@ -16,6 +16,7 @@ import { initializeBrowser } from './utils/browser-utils';
 import { LLMFacade } from './llm-facade';
 import { AuthConfig } from './auth-config';
 import { addTestChimpComment } from './script-utils';
+import { CreditUsageService } from './credit-usage-service';
 
 /**
  * Service for orchestrating Playwright script execution
@@ -23,15 +24,42 @@ import { addTestChimpComment } from './script-utils';
 export class ExecutionService {
   private playwrightService: PlaywrightService;
   private llmFacade: LLMFacade;
+  private creditUsageService: CreditUsageService;
   private maxConcurrentExecutions: number;
   private activeExecutions: Set<Promise<any>> = new Set();
+  private logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
 
   constructor(authConfig?: AuthConfig, backendUrl?: string, maxConcurrentExecutions: number = 10) {
     this.playwrightService = new PlaywrightService();
     this.llmFacade = new LLMFacade(authConfig, backendUrl);
+    this.creditUsageService = new CreditUsageService(authConfig, backendUrl);
     this.maxConcurrentExecutions = maxConcurrentExecutions;
   }
 
+  /**
+   * Set a logger callback for capturing execution logs
+   */
+  setLogger(logger: (message: string, level?: 'log' | 'error' | 'warn') => void): void {
+    this.logger = logger;
+  }
+
+  /**
+   * Log a message using the configured logger or console
+   */
+  private log(message: string, level: 'log' | 'error' | 'warn' = 'log'): void {
+    if (this.logger) {
+      this.logger(message, level);
+    } else {
+      if (level === 'error') {
+        console.error(message);
+      } else if (level === 'warn') {
+        console.warn(message);
+      } else {
+        console.log(message);
+      }
+    }
+  }
+
   /**
    * Initialize the execution service
    */
@@ -39,6 +67,14 @@ export class ExecutionService {
     await this.playwrightService.initialize();
   }
 
+  /**
+   * Set authentication configuration for the service
+   */
+  setAuthConfig(authConfig: AuthConfig): void {
+    this.llmFacade.setAuthConfig(authConfig);
+    this.creditUsageService.setAuthConfig(authConfig);
+  }
+
 
   /**
    * Execute a script with optional AI repair capabilities
@@ -178,7 +214,7 @@ export class ExecutionService {
     const totalAttempts = deflakeRunCount + 1; // Original run + deflake attempts
     let lastError: Error | null = null;
     
-    console.log(`runExactly: deflake_run_count = ${request.deflake_run_count}, totalAttempts = ${totalAttempts}`);
+    this.log(`runExactly: deflake_run_count = ${request.deflake_run_count}, totalAttempts = ${totalAttempts}`);
 
     // Script content should be provided by the caller (TestChimpService)
     // The TestChimpService handles file reading through the appropriate FileHandler
@@ -187,7 +223,7 @@ export class ExecutionService {
     }
 
     for (let attempt = 1; attempt <= totalAttempts; attempt++) {
-      console.log(`Attempting deflake run ${attempt}/${totalAttempts}`);
+      this.log(`Attempting deflake run ${attempt}/${totalAttempts}`);
       const { browser, context, page } = await this.initializeBrowser(request.playwrightConfig, request.headless, request.playwrightConfigFilePath);
 
       try {
@@ -204,7 +240,7 @@ export class ExecutionService {
         };
       } catch (error) {
         lastError = error instanceof Error ? error : new Error('Script execution failed');
-        console.log(`Initial run failed: ${lastError.message}`);
+        this.log(`Initial run failed: ${lastError.message}`);
         
         try {
           await browser.close();
@@ -214,7 +250,7 @@ export class ExecutionService {
 
         // If this is not the last attempt, continue to next attempt
         if (attempt < totalAttempts) {
-          console.log(`Deflaking attempt ${attempt} failed, trying again... (${attempt + 1}/${totalAttempts})`);
+          this.log(`Deflaking attempt ${attempt} failed, trying again... (${attempt + 1}/${totalAttempts})`);
           continue;
         }
       }
@@ -239,7 +275,7 @@ export class ExecutionService {
     }
 
     // First, try runExactly (which includes deflaking if configured)
-    console.log('Attempting runExactly first (with deflaking if configured)...');
+    this.log('Attempting runExactly first (with deflaking if configured)...');
     const runExactlyResult = await this.runExactly(request, startTime, model);
     
     // If runExactly succeeded, return that result
@@ -248,18 +284,18 @@ export class ExecutionService {
     }
 
     // runExactly failed, start AI repair
-    console.log('runExactly failed, starting AI repair process...');
+    this.log('runExactly failed, starting AI repair process...');
 
     try {
         
         // Start browser initialization and script parsing in parallel for faster startup
-        console.log('Initializing repair browser and parsing script...');
+        this.log('Initializing repair browser and parsing script...');
         const [steps, { browser: repairBrowser, context: repairContext, page: repairPage }] = await Promise.all([
           this.parseScriptIntoSteps(request.script, model),
           this.initializeBrowser(request.playwrightConfig, request.headless, request.playwrightConfigFilePath) // Use request.headless (defaults to false/headed)
         ]);
         
-        console.log('Starting AI repair with parsed steps...');
+        this.log('Starting AI repair with parsed steps...');
         const updatedSteps = await this.repairStepsWithAI(steps, repairPage, repairFlexibility, model);
         
         // Always generate the updated script
@@ -272,13 +308,13 @@ export class ExecutionService {
         const hasSuccessfulRepairs = updatedSteps.some(step => step.success);
         
         // Debug: Log step success status
-        console.log('Step success status:', updatedSteps.map((step, index) => `Step ${index + 1}: ${step.success ? 'SUCCESS' : 'FAILED'}`));
-        console.log('All steps successful:', allStepsSuccessful);
-        console.log('Has successful repairs:', hasSuccessfulRepairs);
+        this.log('Step success status: ' + updatedSteps.map((step, index) => `Step ${index + 1}: ${step.success ? 'SUCCESS' : 'FAILED'}`).join(', '));
+        this.log(`All steps successful: ${allStepsSuccessful}`);
+        this.log(`Has successful repairs: ${hasSuccessfulRepairs}`);
         
         // Debug: Log individual step details
         updatedSteps.forEach((step, index) => {
-          console.log(`Step ${index + 1} details: success=${step.success}, description="${step.description}"`);
+          this.log(`Step ${index + 1} details: success=${step.success}, description="${step.description}"`);
         });
         
         // Update file if we have any successful repairs (partial or complete)
@@ -289,6 +325,11 @@ export class ExecutionService {
           // Ensure the final script has the correct TestChimp comment format with repair advice
           const scriptWithRepairAdvice = addTestChimpComment(finalScript, confidenceResponse.advice);
           
+          // Report credit usage for successful AI repair
+          this.creditUsageService.reportAIRepairCredit().catch(error => {
+            console.warn(`Failed to report credit usage for AI repair:`, error);
+          });
+          
           await repairBrowser.close();
           
           return {
@@ -329,14 +370,14 @@ export class ExecutionService {
   private async parseScriptIntoSteps(script: string, model: string): Promise<(ScriptStep & { success?: boolean; error?: string })[]> {
     // First try LLM-based parsing
     try {
-      console.log('Attempting LLM-based script parsing...');
+      this.log('Attempting LLM-based script parsing...');
       const result = await this.llmFacade.parseScriptIntoSteps(script, model);
-      console.log('LLM parsing successful, got', result.length, 'steps');
+      this.log(`LLM parsing successful, got ${result.length} steps`);
       return result;
     } catch (error) {
-      console.log('LLM parsing failed, falling back to code parsing:', error);
+      this.log(`LLM parsing failed, falling back to code parsing: ${error}`);
       const fallbackResult = this.parseScriptIntoStepsFallback(script);
-      console.log('Fallback parsing successful, got', fallbackResult.length, 'steps');
+      this.log(`Fallback parsing successful, got ${fallbackResult.length} steps`);
       return fallbackResult;
     }
   }
@@ -412,26 +453,26 @@ export class ExecutionService {
     let i = 0;
     while (i < updatedSteps.length) {
       const step = updatedSteps[i];
-      console.log(`Loop iteration: i=${i}, step description="${step.description}", total steps=${updatedSteps.length}`);
+      this.log(`Loop iteration: i=${i}, step description="${step.description}", total steps=${updatedSteps.length}`);
       
       try {
         // Try to execute the step directly without context replay
-        console.log(`Attempting Step ${i + 1}: ${step.description}`);
-        console.log(`  Code: ${step.code}`);
+        this.log(`Attempting Step ${i + 1}: ${step.description}`);
+        this.log(`  Code: ${step.code}`);
         await this.executeStepCode(step.code, page);
         step.success = true;
-        console.log(`Step ${i + 1} executed successfully: ${step.description}`);
-        console.log(`Step ${i + 1} success status set to: ${step.success}`);
+        this.log(`Step ${i + 1} executed successfully: ${step.description}`);
+        this.log(`Step ${i + 1} success status set to: ${step.success}`);
         
         // Add this step's code to the execution context for future steps (for variable tracking)
         executionContext += step.code + '\n';
         i++; // Move to next step
       } catch (error) {
-        console.log(`Step ${i + 1} failed: ${step.description}`);
-        console.log(`  Failed code: ${step.code}`);
-        console.log(`  Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
+        this.log(`Step ${i + 1} failed: ${step.description}`);
+        this.log(`  Failed code: ${step.code}`);
+        this.log(`  Error: ${error instanceof Error ? error.message : 'Unknown error'}`);
         if (error instanceof Error && error.stack) {
-          console.log(`  Stack trace: ${error.stack}`);
+          this.log(`  Stack trace: ${error.stack}`);
         }
         step.success = false;
         step.error = this.safeSerializeError(error);
@@ -449,7 +490,7 @@ export class ExecutionService {
         const originalCode = step.code;
         
         for (let attempt = 1; attempt <= maxTries; attempt++) {
-          console.log(`Step ${i + 1} repair attempt ${attempt}/${maxTries}`);
+          this.log(`Step ${i + 1} repair attempt ${attempt}/${maxTries}`);
           
           // Get current page state for AI repair
           const pageInfo = await this.getEnhancedPageInfo(page);
@@ -472,7 +513,7 @@ export class ExecutionService {
           );
 
           if (!repairSuggestion.shouldContinue) {
-            console.log(`AI decided to stop repair at attempt ${attempt}: ${repairSuggestion.reason}`);
+            this.log(`AI decided to stop repair at attempt ${attempt}: ${repairSuggestion.reason}`);
             break;
           }
 
@@ -485,20 +526,14 @@ export class ExecutionService {
               insertAfterIndex: repairSuggestion.action.operation === StepOperation.INSERT ? i - 1 : undefined // For INSERT, insert before current step
             };
             
-            console.log(`🔧 Applying repair action:`, {
-              operation: repairAction.operation,
-              stepIndex: repairAction.stepIndex,
-              insertAfterIndex: repairAction.insertAfterIndex,
-              newStepDescription: repairAction.newStep?.description,
-              newStepCode: repairAction.newStep?.code
-            });
-            console.log(`🔧 Steps array before repair:`, updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`));
+            this.log(`🔧 Applying repair action: ${repairAction.operation} on step ${repairAction.stepIndex}`);
+            this.log(`🔧 Steps array before repair: ${updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`).join(', ')}`);
             
             const result = await this.applyRepairActionInContext(repairAction, updatedSteps, i, page, executionContext, contextVariables);
             
             if (result.success) {
               repairSuccess = true;
-              console.log(`🔧 Steps array after repair:`, updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`));
+              this.log(`🔧 Steps array after repair: ${updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`).join(', ')}`);
               
               // Mark the appropriate step(s) as successful based on operation type
               if (repairAction.operation === StepOperation.MODIFY) {
@@ -507,7 +542,7 @@ export class ExecutionService {
                 step.error = undefined;
                 updatedSteps[i].success = true;
                 updatedSteps[i].error = undefined;
-                console.log(`Step ${i + 1} marked as successful after MODIFY repair`);
+                this.log(`Step ${i + 1} marked as successful after MODIFY repair`);
               } else if (repairAction.operation === StepOperation.INSERT) {
                 // For INSERT: mark the newly inserted step as successful
                 const insertIndex = repairAction.insertAfterIndex !== undefined ? repairAction.insertAfterIndex + 1 : i + 1;
@@ -527,7 +562,7 @@ export class ExecutionService {
                 repairAction.operation === StepOperation.REMOVE ? 
                 `REMOVE: step at index ${repairAction.stepIndex}` :
                 repairAction.operation;
-              console.log(`Step ${i + 1} repair action ${commandInfo} executed successfully on attempt ${attempt}`);
+              this.log(`Step ${i + 1} repair action ${commandInfo} executed successfully on attempt ${attempt}`);
               
               // Update execution context based on the repair action
               if (repairAction.operation === StepOperation.MODIFY && repairAction.newStep) {
@@ -559,21 +594,21 @@ export class ExecutionService {
               // Update step index based on operation
               if (repairAction.operation === StepOperation.INSERT) {
                 // For INSERT: inserted step is already executed
-                console.log(`INSERT operation: current i=${i}, insertAfterIndex=${repairAction.insertAfterIndex}`);
-                console.log(`INSERT: Steps array length before: ${updatedSteps.length}`);
-                console.log(`INSERT: Steps before operation:`, updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`));
+                this.log(`INSERT operation: current i=${i}, insertAfterIndex=${repairAction.insertAfterIndex}`);
+                this.log(`INSERT: Steps array length before: ${updatedSteps.length}`);
+                this.log(`INSERT: Steps before operation: ${updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`).join(', ')}`);
                 
                 if (repairAction.insertAfterIndex !== undefined && repairAction.insertAfterIndex < i) {
                   // If inserting before current position, current step moved down by 1
-                  console.log(`INSERT before current position: incrementing i from ${i} to ${i + 1}`);
+                  this.log(`INSERT before current position: incrementing i from ${i} to ${i + 1}`);
                   i++; // Move to the original step that was pushed to the next position
                 } else {
                   // If inserting at or after current position, stay at current step
-                  console.log(`INSERT at/after current position: keeping i at ${i}`);
+                  this.log(`INSERT at/after current position: keeping i at ${i}`);
                 }
                 
-                console.log(`INSERT: Steps array length after: ${updatedSteps.length}`);
-                console.log(`INSERT: Steps after operation:`, updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`));
+                this.log(`INSERT: Steps array length after: ${updatedSteps.length}`);
+                this.log(`INSERT: Steps after operation: ${updatedSteps.map((s, idx) => `${idx}: "${s.description}" (success: ${s.success})`).join(', ')}`);
               } else if (repairAction.operation === StepOperation.REMOVE) {
                 // For REMOVE: stay at same index since the next step moved to current position
                 // Don't increment i because the array shifted left
@@ -598,9 +633,9 @@ export class ExecutionService {
               repairSuggestion.action.operation === StepOperation.REMOVE ? 
               `REMOVE: step at index ${repairSuggestion.action.stepIndex}` :
               repairSuggestion.action.operation;
-            console.log(`Step ${i + 1} repair attempt ${attempt} failed (${commandInfo}): ${repairErrorMessage}`);
+            this.log(`Step ${i + 1} repair attempt ${attempt} failed (${commandInfo}): ${repairErrorMessage}`);
             if (repairError instanceof Error && repairError.stack) {
-              console.log(`  Repair stack trace: ${repairError.stack}`);
+              this.log(`  Repair stack trace: ${repairError.stack}`);
             }
             
             // Record this attempt in history
@@ -616,7 +651,7 @@ export class ExecutionService {
         }
 
         if (!repairSuccess) {
-          console.log(`Step ${i + 1} failed after ${maxTries} repair attempts`);
+          this.log(`Step ${i + 1} failed after ${maxTries} repair attempts`);
           break;
         }
       }
@@ -855,8 +890,8 @@ export class ExecutionService {
               success: false,
               error: undefined
             };
-            console.log(`INSERT: Inserting step at index ${insertIndex} with description "${newStep.description}"`);
-            console.log(`INSERT: Steps before insertion:`, steps.map((s, i) => `${i}: "${s.description}" (success: ${s.success})`));
+            this.log(`INSERT: Inserting step at index ${insertIndex} with description "${newStep.description}"`);
+            this.log(`INSERT: Steps before insertion: ${steps.map((s, i) => `${i}: "${s.description}" (success: ${s.success})`).join(', ')}`);
             
             // Preserve success status of existing steps before insertion
             const successStatusMap = new Map(steps.map((step, index) => [index, { success: step.success, error: step.error }]));
@@ -877,9 +912,9 @@ export class ExecutionService {
             
             // CRITICAL FIX: Ensure the inserted step doesn't overwrite existing step data
             // The new step should only have its own description, not inherit from existing steps
-            console.log(`INSERT: Final step array after restoration:`, steps.map((s, i) => `${i}: "${s.description}" (success: ${s.success})`));
+            this.log(`INSERT: Final step array after restoration: ${steps.map((s, i) => `${i}: "${s.description}" (success: ${s.success})`).join(', ')}`);
             
-            console.log(`INSERT: Steps after insertion:`, steps.map((s, i) => `${i}: "${s.description}" (success: ${s.success})`));
+            this.log(`INSERT: Steps after insertion: ${steps.map((s, i) => `${i}: "${s.description}" (success: ${s.success})`).join(', ')}`);
             // Test the new step with current page state
             await this.executeStepCode(action.newStep.code, page);
             return { success: true, updatedContext: executionContext + action.newStep.code };

package/src/index.ts

@@ -49,6 +49,8 @@ export class TestChimpService {
     this.llmFacade = new LLMFacade(this.authConfig || undefined, this.backendUrl);
     this.creditUsageService = new CreditUsageService(this.authConfig || undefined, this.backendUrl);
     this.executionService = new ExecutionService(this.authConfig || undefined, this.backendUrl, maxWorkers || 10);
+    // Set the credit usage service for the execution service
+    this.executionService.setAuthConfig(this.authConfig || {} as AuthConfig);
     this.scenarioService = new ScenarioService(maxWorkers || 2, this.fileHandler, this.authConfig || undefined, this.backendUrl);
   }
 
@@ -64,6 +66,9 @@ export class TestChimpService {
     this.executionService = new ExecutionService(this.authConfig, this.backendUrl, 10);
     this.scenarioService = new ScenarioService(2, this.fileHandler, this.authConfig, this.backendUrl);
     
+    // Set auth config for the execution service
+    this.executionService.setAuthConfig(authConfig);
+    
     // Reinitialize the services
     await this.executionService.initialize();
     await this.scenarioService.initialize();
@@ -77,6 +82,20 @@ export class TestChimpService {
     // Recreate services with new backend URL
     this.llmFacade = new LLMFacade(this.authConfig || undefined, this.backendUrl);
     this.creditUsageService = new CreditUsageService(this.authConfig || undefined, this.backendUrl);
+    this.executionService = new ExecutionService(this.authConfig || undefined, this.backendUrl, 10);
+    this.scenarioService = new ScenarioService(2, this.fileHandler, this.authConfig || undefined, this.backendUrl);
+    
+    // Set auth config for the execution service
+    if (this.authConfig) {
+      this.executionService.setAuthConfig(this.authConfig);
+    }
+  }
+
+  /**
+   * Set logger callback for capturing execution logs
+   */
+  setLogger(logger: (message: string, level?: 'log' | 'error' | 'warn') => void): void {
+    this.executionService.setLogger(logger);
   }
 
   /**