testchimp-runner-core 0.0.35 → 0.0.37

package/src/orchestrator/types.ts

@@ -1,291 +0,0 @@
-/**
- * Orchestrator Agent Types
- * Core types for the tool-using, memory-maintaining orchestrator agent
- */
-
-/**
- * Self-reflection guidance for next iteration
- */
-export interface SelfReflection {
-  guidanceForNext: string;  // Free-form text: "Try data-testid selectors instead of text"
-  detectingLoop: boolean;   // Agent signals if it's repeating same approach
-  loopReasoning?: string;   // "Tried text-based selectors 3 times, switching to IDs"
-}
-
-/**
- * Note to future self - iteration-specific memory for tactical continuity
- * More flexible than structured SelfReflection - agent can write freely
- */
-export interface NoteToFutureSelf {
-  fromIteration: number;
-  content: string;  // FREE-FORM - agent writes whatever tactical info it needs for next iteration
-  // Examples:
-  // - "Tried #menu-btn, failed. Will try SVG child next."
-  // - "Plan: Hover over menu to reveal dropdown, then click Profile option."
-  // - "Element loads async. Wait 2s after page load before clicking."
-}
-
-/**
- * Coordinate-based action (fallback when selectors fail)
- * Uses percentage-based positioning for resolution independence
- */
-export interface CoordinateAction {
-  type: 'coordinate';
-  action: 'click' | 'doubleClick' | 'rightClick' | 'hover' | 'drag' | 'fill' | 'scroll';
-  
-  // Primary coordinates as percentages (0.0 to 100.0, 3 decimal precision for ~1px accuracy)
-  xPercent: number;  // 0 = left edge, 100 = right edge
-  yPercent: number;  // 0 = top edge, 100 = bottom edge
-  
-  // For drag actions
-  toXPercent?: number;
-  toYPercent?: number;
-  
-  // For fill actions (click then type)
-  value?: string;
-  
-  // For scroll actions
-  scrollAmount?: number;  // Positive = scroll down, negative = scroll up
-}
-
-/**
- * Journey memory - tracks the agent's journey through the scenario
- */
-export interface JourneyMemory {
-  // Step-by-step history (TEXT only, no screenshots)
-  history: MemoryStep[];
-  
-  // Learnings accumulated throughout journey
-  experiences: string[];
-  
-  // Data extracted and saved for later steps
-  extractedData: Record<string, string>;
-  
-  // Latest note from agent (persists across steps for continuity)
-  latestNote?: NoteToFutureSelf;
-}
-
-/**
- * Individual step in journey history
- */
-export interface MemoryStep {
-  stepNumber: number;
-  iteration?: number;
-  action: string;           // Human-readable: "Filled login form"
-  code: string;             // Executed code: "await page.fill('#email', ...)"
-  result: 'success' | 'failure';
-  observation: string;      // What was observed: "Form submitted, redirected to dashboard"
-  error?: string;           // Error details if failed
-  url: string;              // URL after action
-  timestamp: number;
-}
-
-/**
- * Tool call request from agent
- */
-export interface ToolCall {
-  name: string;
-  params: Record<string, any>;
-}
-
-/**
- * Tool execution result
- */
-export interface ToolResult {
-  success: boolean;
-  data?: any;
-  error?: string;
-  learning?: string;  // For exploratory actions: text-based learning extracted from screenshot
-}
-
-/**
- * Agent decision output
- */
-export interface AgentDecision {
-  // Tool requests (agent can call multiple tools)
-  toolCalls?: ToolCall[];
-  toolReasoning?: string;
-  needsToolResults?: boolean;  // Wait for tool results before proceeding with commands
-  
-  // Command batch (executed sequentially)
-  commands?: string[];  // Plain Playwright commands
-  commandReasoning?: string;
-  
-  // Self-reflection for next iteration
-  selfReflection?: SelfReflection;
-  
-  // Note to future self (NEW - free-form iteration memory)
-  noteToFutureSelf?: string;  // Free-form tactical note for next iteration
-  
-  // Coordinate-based action (NEW - fallback when selectors fail)
-  coordinateAction?: CoordinateAction;
-  
-  // Memory updates
-  memoryUpdate?: {
-    action: string;
-    observation: string;
-    extractedData?: Record<string, any>;
-  };
-  
-  // Learnings to add to experiences
-  experiences?: string[];
-  
-  // Termination decision
-  status: 'complete' | 'stuck' | 'infeasible' | 'continue';
-  statusReasoning: string;
-  reasoning: string;  // Overall reasoning for this iteration
-  
-  // Blocker detection (e.g., unexpected modals, tours, cookie consents)
-  blockerDetected?: {
-    description: string;                   // What's blocking (e.g., "Cookie consent modal")
-    clearingCommands: string[];            // Commands to dismiss/clear it
-  };
-  
-  // Step re-evaluation (last resort after repeated failures)
-  stepReEvaluation?: {
-    detected: boolean;                     // True if agent is questioning current step
-    issue: 'prior_incomplete' | 'already_done' | 'wrong_order' | null;
-    explanation: string;                   // Why agent thinks step order is off
-  };
-  
-  // Meta-learning: Suggested prompt improvements based on journey learnings
-  debugInfo?: {
-    suggestedPromptUpdates?: string;      // Confident suggestions for improving system/user prompts
-    reasoning?: string;                    // Why these updates would help
-  };
-}
-
-/**
- * Context provided to agent each iteration
- */
-export interface AgentContext {
-  // Goals & progress
-  overallGoal: string;
-  currentStepGoal: string;
-  stepNumber: number;
-  totalSteps: number;
-  completedSteps: string[];
-  remainingSteps: string[];
-  
-  // Current state (fresh)
-  currentPageInfo: any;  // PageInfo from getEnhancedPageInfo
-  currentURL: string;
-  
-  // Recent memory (6-7 steps)
-  recentSteps: MemoryStep[];
-  
-  // Learnings & data
-  experiences: string[];
-  extractedData: Record<string, string>;
-  
-  // Note from previous iteration (tactical continuity)
-  noteFromPreviousIteration?: NoteToFutureSelf;
-  
-  // Test data / credentials for exploration
-  testDataPrompt?: string;
-  
-  // SoM (Set-of-Marks) screenshot with visual markers
-  somScreenshot?: string;  // Data URL of screenshot with SoM markers
-  somElementMap?: string;  // Text map of SoM IDs to element details for disambiguation
-  
-  // Repair mode context (undefined for script gen/exploration)
-  priorSteps?: string[];   // Steps completed before current (e.g., ["1. Navigate", "2. Login"])
-  nextSteps?: string[];    // Steps after current (e.g., ["5. Submit", "6. Verify"])
-  
-  // Tool results from this iteration (if any)
-  toolResults?: Record<string, ToolResult>;
-}
-
-/**
- * Exploration mode configuration
- */
-export interface ExplorationMode {
-  enabled: boolean;                    // Whether exploration mode is active
-  explorationPrompt: string;           // Journey-specific focus: "Explore Dashboard and test all widgets"
-  testDataPrompt?: string;             // Test data, credentials context
-  maxExplorationSteps?: number;        // Budget limit (default: 50) - agent can stop earlier
-}
-
-/**
- * Configurable guardrails
- */
-export interface AgentConfig {
-  // Per-step limits
-  maxIterationsPerStep?: number;          // Default: 8
-  maxToolCallsPerIteration?: number;      // Default: 5
-  maxCommandsPerIteration?: number;       // Default: 5
-  maxExploratoryActionsPerIteration?: number;  // Default: 3 (Phase 2)
-  maxExploratoryActionsPerStep?: number;       // Default: 10 (Phase 2)
-  
-  // Scenario-wide limits
-  maxConsecutiveStepFailures?: number;    // Default: 3 (increased from 2 to allow for UI changes)
-  maxTotalIterations?: number;            // Default: 50 (across all steps)
-  continueOnStepFailure?: boolean;        // Default: true (try subsequent steps even if commands fail)
-                                          // NOTE: ALWAYS stops on agent_stuck/infeasible (explicit agent decision)
-  
-  // Memory limits
-  maxExperiences?: number;                // Default: 20
-  maxHistorySize?: number;                // Default: 100
-  recentStepsCount?: number;              // Default: 7
-  
-  // Timeouts
-  commandTimeout?: number;                // Default: 30000 (30s)
-  explorationTimeout?: number;            // Default: 2000 (2s) (Phase 2)
-  
-  // Allowed actions
-  allowedExplorationActions?: string[];   // Default: ['hover', 'click_info', 'click_menu', 'focus'] (Phase 2)
-  allowedDomains?: string[];              // For navigate_to_url validation
-  
-  // Feature flags
-  enableCoordinateMode?: boolean;         // Default: false (experimental - disable until stable)
-  useSoM?: boolean;                       // Default: true (Set-of-Marks visual mode)
-  somUseSomIdBasedCommands?: boolean;     // Default: false (use semantic selectors first)
-  somRestrictCoordinates?: boolean;       // Default: false (if true, strongly discourage coord commands except as absolute last resort)
-  
-  // Exploration mode (NEW)
-  explorationMode?: ExplorationMode;
-}
-
-/**
- * Step execution result from orchestrator
- */
-export interface OrchestratorStepResult {
-  success: boolean;
-  commands: string[];
-  iterations: number;
-  terminationReason?: 'complete' | 'stuck' | 'infeasible' | 'system_limit' | 'agent_stuck';
-  memory: JourneyMemory;
-  error?: string;
-}
-
-/**
- * Default config values
- */
-export const DEFAULT_AGENT_CONFIG: Required<AgentConfig> = {
-  maxIterationsPerStep: 8,
-  maxToolCallsPerIteration: 5,
-  maxCommandsPerIteration: 5,
-  maxExploratoryActionsPerIteration: 3,
-  maxExploratoryActionsPerStep: 10,
-  maxConsecutiveStepFailures: 3,  // Increased from 2
-  maxTotalIterations: 50,
-  continueOnStepFailure: true,  // Try subsequent steps even if one fails
-  maxExperiences: 20,
-  maxHistorySize: 100,
-  recentStepsCount: 7,
-  commandTimeout: 30000,
-  explorationTimeout: 2000,
-  allowedExplorationActions: ['hover', 'click_info', 'click_menu', 'focus'],
-  allowedDomains: [],
-  enableCoordinateMode: false,  // Disabled by default - experimental feature
-  useSoM: true,  // Enabled by default - use Set-of-Marks visual mode
-  somUseSomIdBasedCommands: false,  // Use semantic selectors first
-  somRestrictCoordinates: false,  // Allow coords as valid fallback (for exploration)
-  explorationMode: {
-    enabled: false,
-    explorationPrompt: '',
-    testDataPrompt: undefined,
-    maxExplorationSteps: 50
-  }
-};
-

package/src/playwright-mcp-service.ts

@@ -1,224 +0,0 @@
-import { ScriptResult, PlaywrightConfig } from './types';
-import { initializeBrowser } from './utils/browser-utils';
-
-/**
- * Service for executing Playwright scripts using worker pool
- */
-export class PlaywrightMCPService {
-  private isConnected = false;
-  private logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
-
-  constructor() {
-    // No initialization needed for direct Playwright execution
-  }
-
-  /**
-   * Set a logger callback for capturing execution logs
-   */
-  setLogger(logger: (message: string, level?: 'log' | 'error' | 'warn') => void): void {
-    this.logger = logger;
-  }
-
-  /**
-   * Log a message using the configured logger
-   */
-  private log(message: string, level: 'log' | 'error' | 'warn' = 'log'): void {
-    if (this.logger) {
-      this.logger(message, level);
-    }
-    // No console fallback - logs are routed to consumer
-  }
-
-  /**
-   * Initialize the service
-   */
-  async initialize(): Promise<void> {
-    try {
-      this.log('Initializing Playwright service...');
-      
-      // No specific initialization needed for direct Playwright execution
-      this.isConnected = true;
-      this.log('Playwright service initialized successfully');
-    } catch (error) {
-      throw new Error(`Failed to initialize Playwright service: ${error}`);
-    }
-  }
-
-  /**
-   * Execute a complete job (prescript + script + postscript) using worker pool
-   */
-  async executeJob(prescript: string | undefined, script: string, postscript: string | undefined, config?: PlaywrightConfig): Promise<{
-    success: boolean;
-    results: {
-      prescript?: ScriptResult;
-      script: ScriptResult;
-      postscript?: ScriptResult;
-    };
-    executionTime: number;
-    error?: string;
-  }> {
-    if (!this.isConnected) {
-      throw new Error('Service not initialized');
-    }
-
-    try {
-      // Execute the job directly using Playwright
-      return await this.executeScriptDirectly(prescript, script, postscript, config);
-    } catch (error) {
-      return {
-        success: false,
-        results: {
-          script: { success: false, output: '', error: '', executionTime: 0 }
-        },
-        executionTime: 0,
-        error: error instanceof Error ? error.message : 'Unknown error occurred'
-      };
-    }
-  }
-
-
-
-  /**
-   * Prepare the script content for execution
-   */
-  private prepareScript(script: string, config?: PlaywrightConfig): string {
-    // If the script looks like a test file, return as-is
-    if (script.includes('test(') || script.includes('describe(')) {
-      return script;
-    }
-
-    // If it's a list of Playwright commands, wrap in a test
-    return `
-      test('executed script', async ({ page }) => {
-        ${script}
-      });
-    `;
-  }
-
-  /**
-   * Close the service
-   */
-  /**
-   * Execute script directly using Playwright
-   */
-  private async executeScriptDirectly(
-    prescript: string | undefined, 
-    script: string, 
-    postscript: string | undefined, 
-    config?: PlaywrightConfig
-  ): Promise<{
-    success: boolean;
-    results: {
-      prescript?: ScriptResult;
-      script: ScriptResult;
-      postscript?: ScriptResult;
-    };
-    executionTime: number;
-    error?: string;
-  }> {
-    const startTime = Date.now();
-    
-    // Dynamically import Playwright types
-    const playwright = require('playwright');
-    
-    let browser: any | undefined;
-    let context: any | undefined;
-    let page: any | undefined;
-
-    try {
-      // Use the centralized browser initialization utility
-      const browserInstance = await initializeBrowser(config, undefined, undefined, this.logger);
-      browser = browserInstance.browser;
-      context = browserInstance.context;
-      page = browserInstance.page;
-
-      const results: {
-        prescript?: ScriptResult;
-        script: ScriptResult;
-        postscript?: ScriptResult;
-      } = {
-        script: { success: false, output: '', error: '', executionTime: 0 }
-      };
-
-      // Execute prescript
-      if (prescript) {
-        try {
-          const scriptFunction = new Function('page', 'browser', 'context', `
-            return (async () => {
-              ${prescript}
-            })();
-          `);
-          await scriptFunction(page, browser, context);
-          results.prescript = { success: true, output: 'Prescript executed successfully', error: '', executionTime: 0 };
-        } catch (error: any) {
-          results.prescript = { success: false, output: '', error: error.message, executionTime: 0 };
-        }
-      }
-
-      // Execute main script
-      try {
-        const scriptFunction = new Function('page', 'browser', 'context', `
-          return (async () => {
-            ${script}
-          })();
-        `);
-        await scriptFunction(page, browser, context);
-        results.script = { success: true, output: 'Script executed successfully', error: '', executionTime: 0 };
-      } catch (error: any) {
-        results.script = { success: false, output: '', error: error.message, executionTime: 0 };
-      }
-
-      // Execute postscript
-      if (postscript) {
-        try {
-          const scriptFunction = new Function('page', 'browser', 'context', `
-            return (async () => {
-              ${postscript}
-            })();
-          `);
-          await scriptFunction(page, browser, context);
-          results.postscript = { success: true, output: 'Postscript executed successfully', error: '', executionTime: 0 };
-        } catch (error: any) {
-          results.postscript = { success: false, output: '', error: error.message, executionTime: 0 };
-        }
-      }
-
-      return {
-        success: results.script.success,
-        results,
-        executionTime: Date.now() - startTime
-      };
-
-    } catch (error: any) {
-      return {
-        success: false,
-        results: {
-          script: { success: false, output: '', error: error.message, executionTime: 0 }
-        },
-        executionTime: Date.now() - startTime,
-        error: error.message
-      };
-    } finally {
-      if (browser) {
-        await browser.close();
-      }
-    }
-  }
-
-  async close(): Promise<void> {
-    try {
-      // No cleanup needed for direct Playwright execution
-      this.isConnected = false;
-      this.log('Playwright service closed');
-    } catch (error) {
-      this.log(`Error during shutdown: ${error}`, 'error');
-    }
-  }
-
-  /**
-   * Check if the service is ready
-   */
-  isReady(): boolean {
-    return this.isConnected;
-  }
-}

package/src/progress-reporter.ts

@@ -1,144 +0,0 @@
-/**
- * Progress Reporter Interface
- * Allows consumers to track execution progress (logs, DB writes, etc.)
- */
-
-/**
- * Execution status for a step
- * Matches scriptservice StepExecutionStatus enum
- */
-export enum StepExecutionStatus {
-  SUCCESS = 'SUCCESS_STEP_EXECUTION',
-  FAILURE = 'FAILURE_STEP_EXECUTION',
-  IN_PROGRESS = 'IN_PROGRESS_STEP_EXECUTION'
-}
-
-/**
- * Step progress report - camelCase for TypeScript
- * Structure matches scriptservice's SmartTestExecutionStep / ScriptGenStep
- */
-export interface StepProgress {
-  jobId: string;                      // Job ID for DB keying
-  stepId?: string;                    // Unique step identifier
-  stepNumber: number;                 // Step index (1-based)
-  description: string;                // Step description
-  code?: string;                      // Playwright command executed
-  screenshotDataUrl?: string;         // Screenshot as data URL (data:image/png;base64,...)
-  status: StepExecutionStatus;        // Execution status
-  error?: string;                     // Error message if failed
-  wasRepaired?: boolean;              // For repair mode
-  subActionCount?: number;            // Internal tracking
-  attempt?: number;                   // Internal tracking
-  
-  // Agent transparency (for orchestrator mode)
-  agentIteration?: number;            // Which iteration of the agent loop
-  agentReasoning?: string;            // Agent's reasoning for this iteration
-  agentSelfReflection?: any;          // SelfReflection guidance for next iteration
-  agentExperiences?: string[];        // Learnings from this iteration
-  agentToolsUsed?: string[];          // Tools the agent used
-  agentStatus?: string;               // Agent's status decision
-}
-
-/**
- * Job progress report - camelCase for TypeScript
- */
-export interface JobProgress {
-  jobId: string;                      // Job identifier
-  status: 'started' | 'in_progress' | 'completed' | 'failed';
-  currentStep?: number;               // Current step being executed
-  totalSteps?: number;                // Total steps in scenario
-  testName?: string;                  // Test name
-  script?: string;                    // Generated/repaired script
-  error?: string;                     // Error message if failed
-}
-
-/**
- * Token usage tracking
- */
-export interface TokenUsage {
-  jobId: string;
-  stepNumber?: number;
-  iteration?: number;
-  inputTokens: number;
-  outputTokens: number;
-  includesImage: boolean;
-  model: string;
-  timestamp: number;
-}
-
-/**
- * Additional step info for lifecycle callbacks
- */
-export interface StepInfo {
-  stepId?: string;
-  stepNumber: number;
-  description: string;
-  code?: string;
-}
-
-/**
- * Progress reporter interface for external consumers
- */
-export interface ProgressReporter {
-  /**
-   * Called when a step starts, updates, or completes
-   * - VS Extension/GitHub: Log to console
-   * - Script Service: Write to DB, upload screenshot to GCS
-   */
-  onStepProgress?(progress: StepProgress): Promise<void>;
-  
-  /**
-   * Called when overall job status changes
-   * - VS Extension/GitHub: Log status
-   * - Script Service: Update job in DB
-   */
-  onJobProgress?(progress: JobProgress): Promise<void>;
-  
-  /**
-   * Called when script is generated/updated
-   */
-  onScriptGenerated?(jobId: string, script: string, testName: string): Promise<void>;
-  
-  /**
-   * Called when script is repaired/updated
-   */
-  onScriptRepaired?(jobId: string, originalScript: string, repairedScript: string, confidence: number): Promise<void>;
-  
-  /**
-   * Called when tokens are used (for cost tracking)
-   * - VS Extension/GitHub: Log token usage
-   * - Script Service: Store in DB for analytics
-   */
-  onTokensUsed?(usage: TokenUsage): Promise<void>;
-  
-  /**
-   * Generic logging (for environments that don't need structured progress)
-   */
-  log?(message: string, level?: 'log' | 'error' | 'warn'): void;
-  
-  /**
-   * LIFECYCLE CALLBACKS (optional - used by scriptservice, ignored by local clients)
-   */
-  
-  /**
-   * Called before test execution starts
-   * - Script Service: Initialize browser context, set up DB records
-   * - VS Extension/GitHub: Not used (ignore)
-   */
-  beforeStartTest?(page: any, browser: any, context: any): Promise<void>;
-  
-  /**
-   * Called before each step execution
-   * - Script Service: Update step status to IN_PROGRESS in DB
-   * - VS Extension/GitHub: Not used (ignore)
-   */
-  beforeStepStart?(step: StepInfo, page: any): Promise<void>;
-  
-  /**
-   * Called after test execution completes (success or failure)
-   * - Script Service: Write final status to DB, cleanup resources
-   * - VS Extension/GitHub: Not used (return value is sufficient)
-   */
-  afterEndTest?(status: 'passed' | 'failed', error?: string, page?: any): Promise<void>;
-}
-