testchimp-runner-core 0.0.25 → 0.0.28

package/src/execution-service.ts

@@ -235,14 +235,78 @@ export class ExecutionService {
       throw new Error('Script content is required for execution. The TestChimpService should read the file and provide script content.');
     }
 
+    // Check if we should use existing browser or create new one
+    const useExistingBrowser = !!(request.existingBrowser && request.existingContext && request.existingPage);
+    
+    if (useExistingBrowser) {
+      this.log('Using existing browser/page provided by caller');
+      // Single attempt with existing browser
+      const browser = request.existingBrowser;
+      const context = request.existingContext;
+      const page = request.existingPage;
+      
+      try {
+        // LIFECYCLE: Call beforeStartTest if provided
+        if (this.progressReporter?.beforeStartTest) {
+          await this.progressReporter.beforeStartTest(page, browser, context);
+        }
+        
+        // Execute the script as-is
+        await this.executeScriptContent(request.script, page);
+        
+        // LIFECYCLE: Call afterEndTest on success
+        if (this.progressReporter?.afterEndTest) {
+          await this.progressReporter.afterEndTest('passed', undefined, page);
+        }
+        
+        // Don't close browser - caller owns it
+        
+        return {
+          run_status: 'success',
+          num_deflake_runs: 0,
+          executionTime: Date.now() - startTime
+        };
+      } catch (error) {
+        lastError = error instanceof Error ? error : new Error('Script execution failed');
+        this.log(`Execution failed: ${lastError.message}`);
+        
+        // LIFECYCLE: Call afterEndTest on failure
+        if (this.progressReporter?.afterEndTest) {
+          try {
+            await this.progressReporter.afterEndTest('failed', lastError.message, page);
+          } catch (callbackError) {
+            this.log(`afterEndTest callback failed: ${callbackError}`, 'warn');
+          }
+        }
+        
+        return {
+          run_status: 'failed',
+          num_deflake_runs: 0,
+          executionTime: Date.now() - startTime,
+          error: lastError.message
+        };
+      }
+    }
+
+    // Create our own browser (original behavior)
     for (let attempt = 1; attempt <= totalAttempts; attempt++) {
       this.log(`Attempting deflake run ${attempt}/${totalAttempts}`);
       const { browser, context, page } = await this.initializeBrowser(request.playwrightConfig, request.headless, request.playwrightConfigFilePath);
 
       try {
+        // LIFECYCLE: Call beforeStartTest if provided
+        if (this.progressReporter?.beforeStartTest) {
+          await this.progressReporter.beforeStartTest(page, browser, context);
+        }
+        
         // Execute the script as-is
         await this.executeScriptContent(request.script, page);
         
+        // LIFECYCLE: Call afterEndTest on success
+        if (this.progressReporter?.afterEndTest) {
+          await this.progressReporter.afterEndTest('passed', undefined, page);
+        }
+        
         await browser.close();
         
         // Success! Return immediately
@@ -255,6 +319,15 @@ export class ExecutionService {
         lastError = error instanceof Error ? error : new Error('Script execution failed');
         this.log(`Initial run failed: ${lastError.message}`);
         
+        // LIFECYCLE: Call afterEndTest on failure
+        if (this.progressReporter?.afterEndTest) {
+          try {
+            await this.progressReporter.afterEndTest('failed', lastError.message, page);
+          } catch (callbackError) {
+            this.log(`afterEndTest callback failed: ${callbackError}`, 'warn');
+          }
+        }
+        
         try {
           await browser.close();
         } catch (closeError) {
@@ -287,6 +360,9 @@ export class ExecutionService {
       throw new Error('Script content is required for AI repair. The TestChimpService should read the file and provide script content.');
     }
 
+    // Check if we should use existing browser
+    const useExistingBrowser = !!(request.existingBrowser && request.existingContext && request.existingPage);
+
     // First, try runExactly (which includes deflaking if configured)
     this.log('Attempting runExactly first (with deflaking if configured)...');
     const runExactlyResult = await this.runExactly(request, startTime, model);
@@ -300,16 +376,46 @@ export class ExecutionService {
     this.log('runExactly failed, starting AI repair process...');
 
     try {
+        let repairBrowser, repairContext, repairPage, steps, updatedSteps;
         
-        // Start browser initialization and script parsing in parallel for faster startup
-        this.log('Initializing repair browser and parsing script...');
-        const [steps, { browser: repairBrowser, context: repairContext, page: repairPage }] = await Promise.all([
-          this.parseScriptIntoSteps(request.script, model),
-          this.initializeBrowser(request.playwrightConfig, request.headless, request.playwrightConfigFilePath) // Use request.headless (defaults to false/headed)
-        ]);
-        
-        this.log('Starting AI repair with parsed steps...');
-        const updatedSteps = await this.repairStepsWithAI(steps, repairPage, repairFlexibility, model);
+        if (useExistingBrowser) {
+          // Use existing browser
+          this.log('Using existing browser for AI repair...');
+          repairBrowser = request.existingBrowser;
+          repairContext = request.existingContext;
+          repairPage = request.existingPage;
+          
+          // Parse script into steps
+          steps = await this.parseScriptIntoSteps(request.script, model);
+          
+          // LIFECYCLE: Call beforeStartTest if provided
+          if (this.progressReporter?.beforeStartTest) {
+            await this.progressReporter.beforeStartTest(repairPage, repairBrowser, repairContext);
+          }
+          
+          this.log('Starting AI repair with parsed steps...');
+          updatedSteps = await this.repairStepsWithAI(steps, repairPage, repairFlexibility, model);
+        } else {
+          // Start browser initialization and script parsing in parallel for faster startup
+          this.log('Initializing repair browser and parsing script...');
+          const results = await Promise.all([
+            this.parseScriptIntoSteps(request.script, model),
+            this.initializeBrowser(request.playwrightConfig, request.headless, request.playwrightConfigFilePath) // Use request.headless (defaults to false/headed)
+          ]);
+          
+          steps = results[0];
+          repairBrowser = results[1].browser;
+          repairContext = results[1].context;
+          repairPage = results[1].page;
+          
+          // LIFECYCLE: Call beforeStartTest if provided
+          if (this.progressReporter?.beforeStartTest) {
+            await this.progressReporter.beforeStartTest(repairPage, repairBrowser, repairContext);
+          }
+          
+          this.log('Starting AI repair with parsed steps...');
+          updatedSteps = await this.repairStepsWithAI(steps, repairPage, repairFlexibility, model);
+        }
         
         // Always generate the updated script
         const updatedScript = this.generateUpdatedScript(updatedSteps);
@@ -343,7 +449,23 @@ export class ExecutionService {
             this.log(`Failed to report credit usage for AI repair: ${error}`, 'warn');
           });
           
-          await repairBrowser.close();
+          // LIFECYCLE: Call afterEndTest (partial or complete success)
+          if (this.progressReporter?.afterEndTest) {
+            try {
+              await this.progressReporter.afterEndTest(
+                allStepsSuccessful ? 'passed' : 'failed',
+                allStepsSuccessful ? undefined : 'Partial repair success',
+                repairPage
+              );
+            } catch (callbackError) {
+              this.log(`afterEndTest callback failed: ${callbackError}`, 'warn');
+            }
+          }
+          
+          // Only close browser if we created it (not provided by caller)
+          if (!useExistingBrowser) {
+            await repairBrowser.close();
+          }
           
           return {
             run_status: 'failed', // Original script failed
@@ -356,7 +478,20 @@ export class ExecutionService {
           };
         } else {
           // No successful repairs at all
-          await repairBrowser.close();
+          
+          // LIFECYCLE: Call afterEndTest (complete failure)
+          if (this.progressReporter?.afterEndTest) {
+            try {
+              await this.progressReporter.afterEndTest('failed', 'AI repair could not fix any steps', repairPage);
+            } catch (callbackError) {
+              this.log(`afterEndTest callback failed: ${callbackError}`, 'warn');
+            }
+          }
+          
+          // Only close browser if we created it (not provided by caller)
+          if (!useExistingBrowser) {
+            await repairBrowser.close();
+          }
           
           return {
             run_status: 'failed', // Original script failed
@@ -469,6 +604,18 @@ export class ExecutionService {
       this.log(`Loop iteration: i=${i}, step description="${step.description}", total steps=${updatedSteps.length}`);
       
       try {
+        // LIFECYCLE: Call beforeStepStart if provided
+        if (this.progressReporter?.beforeStepStart) {
+          await this.progressReporter.beforeStepStart(
+            {
+              stepNumber: i + 1,
+              description: step.description,
+              code: step.code
+            },
+            page
+          );
+        }
+        
         // Try to execute the step directly without context replay
         this.log(`Attempting Step ${i + 1}: ${step.description}`);
         this.log(`  Code: ${step.code}`);

package/src/index.ts

@@ -10,9 +10,9 @@ import { ScenarioWorker } from './scenario-worker-class';
 import { PlaywrightMCPService } from './playwright-mcp-service';
 import { LLMFacade } from './llm-facade';
 import { AuthConfig } from './auth-config';
-import { CreditUsageService } from './credit-usage-service';
+import { CreditUsageService, CreditUsageCallback, CreditUsage, CreditUsageReason } from './credit-usage-service';
 
-export { ExecutionService, ScenarioService, ScenarioWorker, PlaywrightMCPService, LLMFacade, CreditUsageService };
+export { ExecutionService, ScenarioService, ScenarioWorker, PlaywrightMCPService, LLMFacade, CreditUsageService, CreditUsageCallback, CreditUsage, CreditUsageReason };
 
 // File handlers
 import { FileHandler, LocalFileHandler, CIFileHandler, NoOpFileHandler } from './file-handler';
@@ -20,12 +20,12 @@ export { FileHandler, LocalFileHandler, CIFileHandler, NoOpFileHandler };
 
 // LLM Provider interfaces
 import { LLMProvider, LLMRequest, LLMResponse } from './llm-provider';
-import { ProgressReporter, StepProgress, JobProgress, StepExecutionStatus } from './progress-reporter';
+import { ProgressReporter, StepProgress, JobProgress, StepExecutionStatus, StepInfo } from './progress-reporter';
 import { BackendProxyLLMProvider } from './providers/backend-proxy-llm-provider';
 import { LocalLLMProvider } from './providers/local-llm-provider';
 
 export { LLMProvider, LLMRequest, LLMResponse };
-export { ProgressReporter, StepProgress, JobProgress, StepExecutionStatus };
+export { ProgressReporter, StepProgress, JobProgress, StepExecutionStatus, StepInfo };
 export { BackendProxyLLMProvider, LocalLLMProvider };
 
 // Orchestrator (tool-using agent)
@@ -80,6 +80,8 @@ export class TestChimpService {
   private backendUrl: string;
   private logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
   private orchestratorOptions?: { useOrchestrator?: boolean; orchestratorConfig?: Partial<AgentConfig>; debugMode?: boolean };
+  private outputChannel?: any;  // Store outputChannel to preserve it across service recreations
+  private creditUsageCallback?: CreditUsageCallback;  // Store credit callback to preserve it across service recreations
 
   constructor(
     fileHandler?: FileHandler,
@@ -88,19 +90,22 @@ export class TestChimpService {
     maxWorkers?: number,
     llmProvider?: LLMProvider,
     progressReporter?: ProgressReporter,
-    orchestratorOptions?: { useOrchestrator?: boolean; orchestratorConfig?: Partial<AgentConfig>; debugMode?: boolean }
+    orchestratorOptions?: { useOrchestrator?: boolean; orchestratorConfig?: Partial<AgentConfig>; debugMode?: boolean },
+    creditUsageCallback?: CreditUsageCallback
   ) {
     this.fileHandler = fileHandler || new NoOpFileHandler();
     this.authConfig = authConfig || null;
     this.backendUrl = backendUrl || 'https://featureservice.testchimp.io'; // Default to production
     this.progressReporter = progressReporter;
     this.orchestratorOptions = orchestratorOptions;
+    this.creditUsageCallback = creditUsageCallback;
     
     // Use provided LLM provider or default to backend proxy (backward compatible)
     this.llmProvider = llmProvider || new BackendProxyLLMProvider(authConfig, backendUrl);
     
     this.playwrightService = new PlaywrightMCPService();
-    this.creditUsageService = new CreditUsageService(this.authConfig || undefined, this.backendUrl);
+    // Pass credit callback to constructor - preserved across recreations via this.creditUsageCallback
+    this.creditUsageService = new CreditUsageService(this.authConfig || undefined, this.backendUrl, this.creditUsageCallback);
     
     // Create services with providers
     this.executionService = new ExecutionService(
@@ -163,6 +168,11 @@ export class TestChimpService {
       this.scenarioService.setLogger(this.logger);
     }
     
+    // Reapply outputChannel if we have one (critical for orchestrator logs)
+    if (this.outputChannel) {
+      this.scenarioService.setOutputChannel(this.outputChannel);
+    }
+    
     // Reinitialize the services
     await this.executionService.initialize();
     await this.scenarioService.initialize();
@@ -183,8 +193,8 @@ export class TestChimpService {
       this.llmProvider.setLogger?.(this.logger);
     }
     
-    // Recreate services with new provider
-    this.creditUsageService = new CreditUsageService(this.authConfig || undefined, this.backendUrl);
+    // Recreate services with new provider (preserve credit callback)
+    this.creditUsageService = new CreditUsageService(this.authConfig || undefined, this.backendUrl, this.creditUsageCallback);
     this.executionService = new ExecutionService(
       this.authConfig || undefined,
       this.backendUrl,
@@ -208,6 +218,11 @@ export class TestChimpService {
       this.executionService.setLogger(this.logger);
       this.scenarioService.setLogger(this.logger);
     }
+    
+    // Reapply outputChannel if we have one (critical for orchestrator logs)
+    if (this.outputChannel) {
+      this.scenarioService.setOutputChannel(this.outputChannel);
+    }
   }
 
   /**
@@ -226,11 +241,22 @@ export class TestChimpService {
    * This enables orchestrator thinking logs to appear in output console
    */
   setOutputChannel(outputChannel: any): void {
+    this.outputChannel = outputChannel;  // Store for future service recreations
     if (typeof this.scenarioService?.setOutputChannel === 'function') {
       this.scenarioService.setOutputChannel(outputChannel);
     }
   }
 
+  /**
+   * Set credit usage callback
+   * Server-side: Use callback to update DB directly (no axios calls)
+   * Client-side: Don't set callback, uses auth for axios calls to backend
+   */
+  setCreditUsageCallback(callback: CreditUsageCallback): void {
+    this.creditUsageCallback = callback;  // Store for future service recreations
+    this.creditUsageService.setCreditUsageCallback(callback);
+  }
+
   /**
    * Log a message using the configured logger
    */
@@ -259,8 +285,26 @@ export class TestChimpService {
   }
 
   // Scenario generation
-  async generateScript(scenario: string, testName?: string, config?: string, model?: string, scenarioFileName?: string): Promise<string> {
-    return this.scenarioService.processScenario(scenario, testName, config, model, scenarioFileName);
+  async generateScript(
+    scenario: string, 
+    testName?: string, 
+    config?: string, 
+    model?: string, 
+    scenarioFileName?: string,
+    existingBrowser?: any,
+    existingContext?: any,
+    existingPage?: any
+  ): Promise<string> {
+    return this.scenarioService.processScenario(
+      scenario, 
+      testName, 
+      config, 
+      model, 
+      scenarioFileName,
+      existingBrowser,
+      existingContext,
+      existingPage
+    );
   }
 
   // Test execution

package/src/progress-reporter.ts

@@ -66,6 +66,16 @@ export interface TokenUsage {
   timestamp: number;
 }
 
+/**
+ * Additional step info for lifecycle callbacks
+ */
+export interface StepInfo {
+  stepId?: string;
+  stepNumber: number;
+  description: string;
+  code?: string;
+}
+
 /**
  * Progress reporter interface for external consumers
  */
@@ -105,5 +115,30 @@ export interface ProgressReporter {
    * Generic logging (for environments that don't need structured progress)
    */
   log?(message: string, level?: 'log' | 'error' | 'warn'): void;
+  
+  /**
+   * LIFECYCLE CALLBACKS (optional - used by scriptservice, ignored by local clients)
+   */
+  
+  /**
+   * Called before test execution starts
+   * - Script Service: Initialize browser context, set up DB records
+   * - VS Extension/GitHub: Not used (ignore)
+   */
+  beforeStartTest?(page: any, browser: any, context: any): Promise<void>;
+  
+  /**
+   * Called before each step execution
+   * - Script Service: Update step status to IN_PROGRESS in DB
+   * - VS Extension/GitHub: Not used (ignore)
+   */
+  beforeStepStart?(step: StepInfo, page: any): Promise<void>;
+  
+  /**
+   * Called after test execution completes (success or failure)
+   * - Script Service: Write final status to DB, cleanup resources
+   * - VS Extension/GitHub: Not used (return value is sufficient)
+   */
+  afterEndTest?(status: 'passed' | 'failed', error?: string, page?: any): Promise<void>;
 }
 

package/src/prompts.ts

@@ -757,12 +757,12 @@ ${script}
 
 YOUR TASK (MINOR ADJUSTMENTS ONLY):
 1. Remove duplicate/redundant expect() assertions (e.g., same assertion repeated twice)
-2. Remove duplicate step comments without code
-3. Fix obvious formatting issues (inconsistent spacing, etc.)
-4. Consolidate multiple identical assertions into one
-5. Remove any obviously redundant waits or checks
+2. Fix obvious formatting issues (inconsistent spacing, etc.)
+3. Consolidate multiple identical assertions into one
+4. Remove any obviously redundant waits or checks
 
 DO NOT:
+- Remove step comments (e.g., "// Step 1: ..." or "// Navigate to...") - these are important for readability
 - Change the test logic or flow
 - Remove legitimate assertions
 - Restructure the code

package/src/scenario-service.ts

@@ -111,16 +111,25 @@ export class ScenarioService extends EventEmitter {
     });
     
     this.workers.push(worker);
-    this.log(`Scenario worker initialized${this.useOrchestrator ? ' (Orchestrator Mode)' : ''} with session: ${worker['sessionId']}`);
+    // Internal initialization - no need to log worker details
   }
 
   async initialize(): Promise<void> {
     // Wait for workers to be initialized
     await this.initializeWorkers();
-    this.log('Scenario service initialized');
+    // Internal initialization - consumer doesn't need to see this
   }
 
-  processScenario(scenario: string, testName?: string, config?: PlaywrightConfig, model?: string, scenarioFileName?: string): string {
+  processScenario(
+    scenario: string, 
+    testName?: string, 
+    config?: PlaywrightConfig, 
+    model?: string, 
+    scenarioFileName?: string,
+    existingBrowser?: any,
+    existingContext?: any,
+    existingPage?: any
+  ): string {
     const jobId = `scenario_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`;
     
     // Add job to queue
@@ -130,7 +139,10 @@ export class ScenarioService extends EventEmitter {
       testName,
       playwrightConfig: config,
       model,
-      scenarioFileName
+      scenarioFileName,
+      existingBrowser,
+      existingContext,
+      existingPage
     };
 
     this.jobQueue.push(job);