testchimp-runner-core 0.0.1

package/src/scenario-worker-class.ts

@@ -0,0 +1,330 @@
+import { Browser, Page, BrowserContext } from 'playwright';
+import fs from 'fs';
+import path from 'path';
+import { getEnhancedPageInfo } from './utils/page-info-utils';
+import { initializeBrowser } from './utils/browser-utils';
+import { LLMFacade } from './llm-facade';
+import { ScenarioRunJob, ScenarioResponse, ScenarioStep } from './types';
+import { FileHandler } from './file-handler';
+import { AuthConfig } from './auth-config';
+import { generateTestScript } from './script-utils';
+
+// Define a simple logging interface for compatibility
+interface OutputChannel {
+  appendLine: (text: string) => void;
+}
+
+// Legacy interface for backward compatibility
+interface ScenarioJob {
+  id: string;
+  scenario: string;
+  testName?: string;
+  playwrightConfig?: string;
+  model?: string;
+  scenarioFileName?: string;
+}
+
+
+const MAX_RETRIES_PER_STEP = 2;
+
+export class ScenarioWorker {
+  private initialized = false;
+  private sessionId: string | null = null;
+  private llmFacade: LLMFacade;
+  private fileHandler?: FileHandler;
+  private outputChannel?: OutputChannel;
+
+  constructor(fileHandler?: FileHandler, authConfig?: AuthConfig, outputChannel?: OutputChannel) {
+    this.llmFacade = new LLMFacade(authConfig);
+    this.fileHandler = fileHandler;
+    this.outputChannel = outputChannel;
+  }
+
+  private log(message: string): void {
+    console.log(message);
+    if (this.outputChannel) {
+      this.outputChannel.appendLine(`[ScenarioWorker] ${message}`);
+    }
+  }
+
+  private logError(message: string): void {
+    console.error(message);
+    if (this.outputChannel) {
+      this.outputChannel.appendLine(`[ScenarioWorker] ERROR: ${message}`);
+    }
+  }
+
+  async initialize(): Promise<void> {
+    try {
+      this.log('Initializing Scenario worker...');
+      this.sessionId = `scenario_worker_${Date.now()}`;
+      this.initialized = true;
+      this.log(`Scenario worker initialized with session: ${this.sessionId}`);
+    } catch (error) {
+      this.logError(`Scenario worker initialization error: ${error}`);
+      throw error;
+    }
+  }
+
+  async processScenarioJob(job: ScenarioJob): Promise<ScenarioResponse> {
+    if (!this.initialized) {
+      throw new Error('Scenario worker not initialized');
+    }
+
+    const startTime = Date.now();
+    const steps: ScenarioStep[] = [];
+    let generatedScript = '';
+    let scriptPath: string | undefined;
+    let browser: Browser | undefined;
+    let context: BrowserContext | undefined;
+    let page: Page | undefined;
+    let overallSuccess = true;
+
+    try {
+      // 1. Break down scenario into steps using LLM
+      const scenarioSteps = await this.llmFacade.breakdownScenario(job.scenario, job.model);
+      steps.push(...scenarioSteps);
+
+      // 2. Start a new browser session using centralized utility
+      // Default to headed mode (headless: false) for better debugging
+      const browserInstance = await initializeBrowser(job.playwrightConfig, false);
+      browser = browserInstance.browser;
+      context = browserInstance.context;
+      page = browserInstance.page;
+      
+      // Set reasonable timeout for all operations
+      page.setDefaultTimeout(5000); // 5 seconds
+
+      let previousSteps: ScenarioStep[] = [];
+      let lastError: string | undefined;
+
+      // 3. Execute each step
+      for (let i = 0; i < steps.length; i++) {
+        const step = steps[i];
+        step.stepNumber = i + 1;
+        step.retryCount = 0;
+
+        let stepSuccess = false;
+        let stepOutput = '';
+        let stepError: string | undefined;
+
+        for (let attempt = 0; attempt <= MAX_RETRIES_PER_STEP; attempt++) {
+          let currentAttemptCommand: string | undefined;
+          let currentAttemptSuccess = false;
+          let currentAttemptError: string | undefined;
+          const attemptTimestamp = Date.now();
+
+          try {
+            this.log(`Attempt ${attempt + 1} for step: ${step.description}`);
+
+            // Get current page state using Playwright's accessibility tree
+            const domSnapshot = {
+              url: page.url(),
+              title: await page.title(),
+              accessibilityTree: await page.accessibility.snapshot()
+            };
+
+            // Generate Playwright command using LLM
+            const pageInfo = await getEnhancedPageInfo(domSnapshot);
+            const command = await this.llmFacade.generatePlaywrightCommand(step.description, pageInfo, previousSteps, lastError, step, job.model);
+
+            if (!command) {
+              throw new Error('LLM failed to generate a Playwright command.');
+            }
+
+            step.playwrightCommand = command;
+            currentAttemptCommand = command;
+            this.log(`  Command: ${command}`);
+
+            // Execute the command
+            await this.executePlaywrightCommand(page, browser, context, command);
+
+            stepSuccess = true;
+            currentAttemptSuccess = true;
+            stepOutput = `Executed: ${command}`;
+            stepError = undefined;
+            this.log(`  ✅ SUCCESS: ${command}`);
+            break; // Step successful, move to next scenario step
+            } catch (error: any) {
+              stepError = error instanceof Error ? error.message : String(error);
+              currentAttemptError = stepError;
+              console.error(`  ❌ FAILED (attempt ${attempt + 1}): ${stepError}`);
+              console.error(`  Command attempted: ${currentAttemptCommand || 'N/A'}`);
+              step.retryCount++;
+              
+              // Only update lastError if this is the final attempt
+              if (attempt === MAX_RETRIES_PER_STEP) {
+                lastError = stepError;
+              }
+              
+              // If this is the last attempt, mark as failed and move on
+              if (attempt === MAX_RETRIES_PER_STEP) {
+                stepSuccess = false;
+                stepOutput = `Failed after ${MAX_RETRIES_PER_STEP + 1} attempts.`;
+                overallSuccess = false;
+                console.error(`  🚫 STEP FAILED after ${MAX_RETRIES_PER_STEP + 1} attempts`);
+                break; // Exit retry loop
+              }
+            } finally {
+              if (!step.attempts) {
+                step.attempts = [];
+              }
+              step.attempts.push({
+                attemptNumber: attempt + 1,
+                command: currentAttemptCommand,
+                success: currentAttemptSuccess,
+                error: currentAttemptError,
+                timestamp: attemptTimestamp
+              });
+            }
+          }
+
+        step.success = stepSuccess;
+        step.error = stepError;
+        previousSteps.push(step);
+      }
+
+      // Generate test name if not provided
+      const testName = job.testName || await this.llmFacade.generateTestName(job.scenario, job.model);
+
+      // Generate preferred filename
+      let preferredFileName: string;
+      if (testName && testName !== 'test') {
+        // Use the generated test name
+        const sanitizedName = testName
+          .replace(/[^a-zA-Z0-9\s-_]/g, '') // Remove special characters except spaces, hyphens, underscores
+          .replace(/\s+/g, '_') // Replace spaces with underscores
+          .replace(/_+/g, '_') // Replace multiple underscores with single underscore
+          .replace(/^_|_$/g, '') // Remove leading/trailing underscores
+          .toLowerCase();
+        preferredFileName = `${sanitizedName}.spec.js`;
+      } else {
+        // Use scenario file name if no meaningful test name was generated
+        const scenarioFileName = job.scenarioFileName || 'scenario';
+        const baseName = scenarioFileName.replace(/\.[^/.]+$/, ''); // Remove extension
+        const sanitizedName = baseName
+          .replace(/[^a-zA-Z0-9\s-_]/g, '') // Remove special characters except spaces, hyphens, underscores
+          .replace(/\s+/g, '_') // Replace spaces with underscores
+          .replace(/_+/g, '_') // Replace multiple underscores with single underscore
+          .replace(/^_|_$/g, '') // Remove leading/trailing underscores
+          .toLowerCase();
+        preferredFileName = `${sanitizedName}.spec.js`;
+      }
+
+      // Generate clean script with TestChimp comment and code
+      generatedScript = generateTestScript(testName, steps);
+
+      // Generate detailed execution log
+      const logLines: string[] = [];
+      logLines.push(`# Scenario Execution Log`);
+      logLines.push(`Job ID: ${job.id}`);
+      logLines.push(`Scenario: ${job.scenario}`);
+      logLines.push(`Start Time: ${new Date(startTime).toISOString()}`);
+      logLines.push(`End Time: ${new Date().toISOString()}`);
+      logLines.push(`Total Execution Time: ${Date.now() - startTime}ms`);
+      logLines.push(`Overall Success: ${overallSuccess ? 'YES' : 'NO'}`);
+      logLines.push(``);
+      
+      for (const step of steps) {
+        logLines.push(`## Step ${step.stepNumber}: ${step.description}`);
+        logLines.push(`Status: ${step.success ? 'SUCCESS' : 'FAILED'}`);
+        logLines.push(`Retry Count: ${step.retryCount || 0}`);
+        
+        if (step.playwrightCommand) {
+          logLines.push(`Final Command: ${step.playwrightCommand}`);
+        }
+        
+        if (step.error) {
+          logLines.push(`Final Error: ${step.error}`);
+        }
+
+        if (step.attempts && step.attempts.length > 0) {
+          logLines.push(`### Attempts:`);
+          for (const attempt of step.attempts) {
+            logLines.push(`- Attempt ${attempt.attemptNumber}:`);
+            logLines.push(`  Command: ${attempt.command || 'N/A'}`);
+            logLines.push(`  Success: ${attempt.success ? 'YES' : 'NO'}`);
+            if (attempt.error) {
+              logLines.push(`  Error: ${attempt.error}`);
+            }
+            logLines.push(`  Timestamp: ${new Date(attempt.timestamp).toISOString()}`);
+          }
+        }
+        
+        logLines.push(``);
+      }
+      
+      const executionLog = logLines.join('\n');
+
+      return {
+        success: overallSuccess,
+        steps,
+        generatedScript,
+        executionLog,
+        executionTime: Date.now() - startTime,
+        testName,
+        preferredFileName
+      };
+
+    } catch (error: any) {
+      overallSuccess = false;
+      console.error('Overall scenario processing error:', error);
+      return {
+        success: false,
+        steps,
+        generatedScript,
+        executionLog: `# Scenario Execution Log\nJob ID: ${job.id}\nScenario: ${job.scenario}\nError: ${error instanceof Error ? error.message : 'Unknown error during scenario processing'}`,
+        executionTime: Date.now() - startTime,
+        testName: job.testName || 'test',
+        preferredFileName: 'test.spec.js',
+        error: error instanceof Error ? error.message : 'Unknown error during scenario processing'
+      };
+    } finally {
+      if (browser) {
+        await browser.close();
+      }
+    }
+  }
+
+
+
+
+
+
+  private async executePlaywrightCommand(
+    page: Page, 
+    browser: Browser, 
+    context: BrowserContext, 
+    command: string
+  ): Promise<void> {
+    // Set reasonable timeouts
+    page.setDefaultTimeout(5000); // 5 seconds
+
+    try {
+      // Execute command directly without validation
+      const commandFunction = new Function('page', 'browser', 'context', 'expect', `
+        return (async () => {
+          try {
+            ${command}
+          } catch (error) {
+            console.error('Command execution error:', error);
+            throw error;
+          }
+        })();
+      `);
+
+      await commandFunction(page, browser, context, require('@playwright/test').expect);
+      
+    } finally {
+      // Reset to default timeout
+      page.setDefaultTimeout(10000); // Reset to default 10 seconds
+    }
+  }
+
+
+
+  async cleanup(): Promise<void> {
+    this.initialized = false;
+    this.sessionId = null;
+  }
+}

package/src/script-utils.ts

@@ -0,0 +1,109 @@
+/**
+ * Script Generation Utilities
+ * 
+ * This module provides utilities for generating and formatting test scripts
+ * with TestChimp-specific markers and comments.
+ */
+
+/**
+ * TestChimp managed test comment that should be added to all generated scripts
+ */
+export const TESTCHIMP_MANAGED_COMMENT = `/*
+
+This is a TestChimp Managed Test.
+
+*/`;
+
+/**
+ * Generates TestChimp managed test comment with optional repair advice
+ * @param repairAdvice Optional repair advice to include in the comment
+ * @returns The complete comment block
+ */
+export function generateTestChimpComment(repairAdvice?: string): string {
+  if (repairAdvice) {
+    return `/*
+
+This is a TestChimp Managed Test.
+
+Repair Advice:
+${repairAdvice}
+
+*/`;
+  }
+  return TESTCHIMP_MANAGED_COMMENT;
+}
+
+/**
+ * Adds the TestChimp managed test comment to the beginning of a script
+ * @param script The original script content
+ * @param repairAdvice Optional repair advice to include in the comment
+ * @returns The script with TestChimp comment prepended
+ */
+export function addTestChimpComment(script: string, repairAdvice?: string): string {
+  // If the script already has the TestChimp comment, update it with repair advice if provided
+  if (script.includes('This is a TestChimp Managed Test')) {
+    if (repairAdvice) {
+      // Replace existing comment with new one that includes repair advice
+      const commentRegex = /\/\*[\s\S]*?This is a TestChimp Managed Test\.[\s\S]*?\*\//;
+      const newComment = generateTestChimpComment(repairAdvice);
+      return script.replace(commentRegex, newComment);
+    }
+    return script;
+  }
+
+  // Add the comment at the beginning of the script
+  const comment = generateTestChimpComment(repairAdvice);
+  return `${comment}\n\n${script}`;
+}
+
+/**
+ * Generates a complete test script with TestChimp comment, imports, and test structure
+ * @param testName The name of the test
+ * @param steps Array of test steps with descriptions and commands
+ * @param includeTestChimpComment Whether to include the TestChimp managed test comment
+ * @param repairAdvice Optional repair advice to include in the comment
+ * @returns The complete test script
+ */
+export function generateTestScript(
+  testName: string, 
+  steps: Array<{ stepNumber: number; description: string; playwrightCommand?: string; success?: boolean }>,
+  includeTestChimpComment: boolean = true,
+  repairAdvice?: string
+): string {
+  const scriptLines: string[] = [];
+  
+  // Add TestChimp comment if requested
+  if (includeTestChimpComment) {
+    const comment = generateTestChimpComment(repairAdvice);
+    scriptLines.push(comment);
+    scriptLines.push('');
+  }
+  
+  // Add imports
+  scriptLines.push(`import { test, expect } from '@playwright/test';`);
+  
+  // Add test structure
+  scriptLines.push(`test('${testName.replace(/'/g, "\\'")}', async ({ page, browser, context }) => {`);
+  
+  // Add steps
+  for (const step of steps) {
+    const status = step.success === false ? ' [FAILED]' : '';
+    scriptLines.push(`  // Step ${step.stepNumber}: ${step.description}${status}`);
+    if (step.playwrightCommand && step.success !== false) {
+      scriptLines.push(`  ${step.playwrightCommand}`);
+    }
+  }
+  
+  scriptLines.push(`});`);
+  
+  return scriptLines.join('\n');
+}
+
+/**
+ * Checks if a script is a TestChimp managed test
+ * @param script The script content to check
+ * @returns True if the script contains the TestChimp managed test comment
+ */
+export function isTestChimpManagedTest(script: string): boolean {
+  return script.includes('This is a TestChimp Managed Test');
+}

package/src/types.ts

@@ -0,0 +1,202 @@
+// ============================================================================
+// CORE TYPES
+// ============================================================================
+
+/**
+ * Playwright MCP configuration - JavaScript config file content (playwright.config.js)
+ */
+export type PlaywrightConfig = string;
+
+// ============================================================================
+// SCRIPT EXECUTION TYPES
+// ============================================================================
+
+/**
+ * Request structure for the Playwright script executor
+ */
+export interface PlaywrightExecutionRequest {
+  /** Main Playwright script content */
+  script: string;
+  /** Optional pre-script to run before the main script */
+  prescript?: string;
+  /** Optional post-script to run after the main script */
+  postscript?: string;
+  /** Playwright configuration file content */
+  playwrightConfig: string;
+  /** Optional GPT model to use for AI operations */
+  model?: string;
+}
+
+/**
+ * Response structure for the Playwright script executor
+ */
+export interface PlaywrightExecutionResponse {
+  /** Whether the execution was successful */
+  success: boolean;
+  /** Execution results from each script phase */
+  results: {
+    prescript?: ScriptResult;
+    script: ScriptResult;
+    postscript?: ScriptResult;
+  };
+  /** Overall execution time in milliseconds */
+  executionTime: number;
+  /** Any errors that occurred during execution */
+  error?: string;
+}
+
+/**
+ * Individual script execution result
+ */
+export interface ScriptResult {
+  /** Whether this specific script executed successfully */
+  success: boolean;
+  /** Output from the script execution */
+  output: string;
+  /** Any errors from this script */
+  error?: string;
+  /** Execution time for this script in milliseconds */
+  executionTime: number;
+}
+
+// ============================================================================
+// SCENARIO EXECUTION TYPES
+// ============================================================================
+
+/**
+ * Scenario execution request
+ */
+export interface ScenarioRequest {
+  scenario: string;
+  testName?: string;
+  playwrightConfig?: PlaywrightConfig;
+  model?: string;
+}
+
+/**
+ * Scenario execution job for worker queue
+ */
+export interface ScenarioRunJob {
+  id: string;
+  scenario: string;
+  testName?: string;
+  playwrightConfig?: PlaywrightConfig;
+  model?: string;
+  scenarioFileName?: string;
+}
+
+/**
+ * Scenario execution response
+ */
+export interface ScenarioResponse {
+  success: boolean;
+  steps: ScenarioStep[];
+  generatedScript: string;
+  executionLog: string;
+  executionTime: number;
+  testName?: string;
+  preferredFileName?: string;
+  error?: string;
+}
+
+/**
+ * Individual scenario step
+ */
+export interface ScenarioStep {
+  stepNumber: number;
+  description: string;
+  playwrightCommand?: string;
+  success?: boolean;
+  error?: string;
+  retryCount?: number;
+  attempts?: Array<{
+    attemptNumber: number;
+    command?: string;
+    success: boolean;
+    error?: string;
+    timestamp: number;
+  }>;
+}
+
+/**
+ * Legacy scenario job interface (for backward compatibility)
+ */
+export interface ScenarioJob {
+  id: string;
+  scenario: string;
+  config?: PlaywrightConfig;
+  resolve: (result: ScenarioResponse) => void;
+  reject: (error: Error) => void;
+}
+
+// ============================================================================
+// AI REPAIR TYPES
+// ============================================================================
+
+/**
+ * Execution mode for script execution
+ */
+export enum ExecutionMode {
+  RUN_EXACTLY = 'RUN_EXACTLY',
+  RUN_WITH_AI_REPAIR = 'RUN_WITH_AI_REPAIR'
+}
+
+/**
+ * Script execution request with AI repair capabilities
+ */
+export interface ScriptExecutionRequest {
+  script?: string; // Optional if scriptFilePath is provided
+  scriptFilePath?: string; // Path to script file (alternative to script content)
+  mode: ExecutionMode;
+  repair_flexibility?: number; // 0-5, defaults to 3
+  playwrightConfig?: PlaywrightConfig;
+  playwrightConfigFilePath?: string; // Path to playwright config file (alternative to playwrightConfig content)
+  model?: string;
+  headless?: boolean; // defaults to false (headed)
+  deflake_run_count?: number; // defaults to 1
+}
+
+/**
+ * Script execution response with repair information
+ */
+export interface ScriptExecutionResponse {
+  run_status: 'success' | 'failed';
+  repair_status?: 'success' | 'failed' | 'partial';
+  repair_confidence?: number; // 0-5
+  repair_advice?: string;
+  updated_script?: string;
+  executionTime: number;
+  num_deflake_runs?: number; // Number of deflaking runs made (excluding original run)
+  error?: string;
+}
+
+/**
+ * Individual script step for AI repair
+ */
+export interface ScriptStep {
+  description: string;
+  code: string;
+  success?: boolean;
+  error?: string;
+}
+
+/**
+ * Step operation types for AI repair
+ */
+export enum StepOperation {
+  MODIFY = 'MODIFY',
+  INSERT = 'INSERT',
+  REMOVE = 'REMOVE'
+}
+
+/**
+ * Step repair action
+ */
+export interface StepRepairAction {
+  operation: StepOperation;
+  stepIndex?: number; // For MODIFY and REMOVE operations
+  newStep?: ScriptStep; // For MODIFY and INSERT operations
+  insertAfterIndex?: number; // For INSERT operation
+}
+
+// Repair suggestion and confidence interfaces are now in llm-facade.ts