testchimp-runner-core 0.0.21 → 0.0.23

package/src/orchestrator/types.ts

@@ -0,0 +1,200 @@
+/**
+ * Orchestrator Agent Types
+ * Core types for the tool-using, memory-maintaining orchestrator agent
+ */
+
+/**
+ * Self-reflection guidance for next iteration
+ */
+export interface SelfReflection {
+  guidanceForNext: string;  // Free-form text: "Try data-testid selectors instead of text"
+  detectingLoop: boolean;   // Agent signals if it's repeating same approach
+  loopReasoning?: string;   // "Tried text-based selectors 3 times, switching to IDs"
+}
+
+/**
+ * Journey memory - tracks the agent's journey through the scenario
+ */
+export interface JourneyMemory {
+  // Step-by-step history (TEXT only, no screenshots)
+  history: MemoryStep[];
+  
+  // Learnings accumulated throughout journey
+  experiences: string[];
+  
+  // Data extracted and saved for later steps
+  extractedData: Record<string, string>;
+}
+
+/**
+ * Individual step in journey history
+ */
+export interface MemoryStep {
+  stepNumber: number;
+  iteration?: number;
+  action: string;           // Human-readable: "Filled login form"
+  code: string;             // Executed code: "await page.fill('#email', ...)"
+  result: 'success' | 'failure';
+  observation: string;      // What was observed: "Form submitted, redirected to dashboard"
+  error?: string;           // Error details if failed
+  url: string;              // URL after action
+  timestamp: number;
+}
+
+/**
+ * Tool call request from agent
+ */
+export interface ToolCall {
+  name: string;
+  params: Record<string, any>;
+}
+
+/**
+ * Tool execution result
+ */
+export interface ToolResult {
+  success: boolean;
+  data?: any;
+  error?: string;
+  learning?: string;  // For exploratory actions: text-based learning extracted from screenshot
+}
+
+/**
+ * Agent decision output
+ */
+export interface AgentDecision {
+  // Tool requests (agent can call multiple tools)
+  toolCalls?: ToolCall[];
+  toolReasoning?: string;
+  needsToolResults?: boolean;  // Wait for tool results before proceeding with commands
+  
+  // Command batch (executed sequentially)
+  commands?: string[];
+  commandReasoning?: string;
+  
+  // Self-reflection for next iteration
+  selfReflection?: SelfReflection;
+  
+  // Memory updates
+  memoryUpdate?: {
+    action: string;
+    observation: string;
+    extractedData?: Record<string, any>;
+  };
+  
+  // Learnings to add to experiences
+  experiences?: string[];
+  
+  // Termination decision
+  status: 'complete' | 'stuck' | 'infeasible' | 'continue';
+  statusReasoning: string;
+  reasoning: string;  // Overall reasoning for this iteration
+  
+  // Blocker detection (e.g., unexpected modals, tours, cookie consents)
+  blockerDetected?: {
+    description: string;                   // What's blocking (e.g., "Cookie consent modal")
+    clearingCommands: string[];            // Commands to dismiss/clear it
+  };
+  
+  // Step re-evaluation (last resort after repeated failures)
+  stepReEvaluation?: {
+    detected: boolean;                     // True if agent is questioning current step
+    issue: 'prior_incomplete' | 'already_done' | 'wrong_order' | null;
+    explanation: string;                   // Why agent thinks step order is off
+  };
+}
+
+/**
+ * Context provided to agent each iteration
+ */
+export interface AgentContext {
+  // Goals & progress
+  overallGoal: string;
+  currentStepGoal: string;
+  stepNumber: number;
+  totalSteps: number;
+  completedSteps: string[];
+  remainingSteps: string[];
+  
+  // Current state (fresh)
+  currentPageInfo: any;  // PageInfo from getEnhancedPageInfo
+  currentURL: string;
+  
+  // Recent memory (6-7 steps)
+  recentSteps: MemoryStep[];
+  
+  // Learnings & data
+  experiences: string[];
+  extractedData: Record<string, string>;
+  
+  // Self-reflection from previous iteration
+  previousIterationGuidance?: SelfReflection;
+  
+  // Tool results from this iteration (if any)
+  toolResults?: Record<string, ToolResult>;
+}
+
+/**
+ * Configurable guardrails
+ */
+export interface AgentConfig {
+  // Per-step limits
+  maxIterationsPerStep?: number;          // Default: 8
+  maxToolCallsPerIteration?: number;      // Default: 5
+  maxCommandsPerIteration?: number;       // Default: 5
+  maxExploratoryActionsPerIteration?: number;  // Default: 3 (Phase 2)
+  maxExploratoryActionsPerStep?: number;       // Default: 10 (Phase 2)
+  
+  // Scenario-wide limits
+  maxConsecutiveStepFailures?: number;    // Default: 3 (increased from 2 to allow for UI changes)
+  maxTotalIterations?: number;            // Default: 50 (across all steps)
+  continueOnStepFailure?: boolean;        // Default: true (try subsequent steps even if commands fail)
+                                          // NOTE: ALWAYS stops on agent_stuck/infeasible (explicit agent decision)
+  
+  // Memory limits
+  maxExperiences?: number;                // Default: 20
+  maxHistorySize?: number;                // Default: 100
+  recentStepsCount?: number;              // Default: 7
+  
+  // Timeouts
+  commandTimeout?: number;                // Default: 30000 (30s)
+  explorationTimeout?: number;            // Default: 2000 (2s) (Phase 2)
+  
+  // Allowed actions
+  allowedExplorationActions?: string[];   // Default: ['hover', 'click_info', 'click_menu', 'focus'] (Phase 2)
+  allowedDomains?: string[];              // For navigate_to_url validation
+}
+
+/**
+ * Step execution result from orchestrator
+ */
+export interface OrchestratorStepResult {
+  success: boolean;
+  commands: string[];
+  iterations: number;
+  terminationReason?: 'complete' | 'stuck' | 'infeasible' | 'system_limit' | 'agent_stuck';
+  memory: JourneyMemory;
+  error?: string;
+}
+
+/**
+ * Default config values
+ */
+export const DEFAULT_AGENT_CONFIG: Required<AgentConfig> = {
+  maxIterationsPerStep: 8,
+  maxToolCallsPerIteration: 5,
+  maxCommandsPerIteration: 5,
+  maxExploratoryActionsPerIteration: 3,
+  maxExploratoryActionsPerStep: 10,
+  maxConsecutiveStepFailures: 3,  // Increased from 2
+  maxTotalIterations: 50,
+  continueOnStepFailure: true,  // Try subsequent steps even if one fails
+  maxExperiences: 20,
+  maxHistorySize: 100,
+  recentStepsCount: 7,
+  commandTimeout: 30000,
+  explorationTimeout: 2000,
+  allowedExplorationActions: ['hover', 'click_info', 'click_menu', 'focus'],
+  allowedDomains: []
+};
+

package/src/playwright-mcp-service.ts

@@ -6,21 +6,39 @@ import { initializeBrowser } from './utils/browser-utils';
  */
 export class PlaywrightMCPService {
   private isConnected = false;
+  private logger?: (message: string, level?: 'log' | 'error' | 'warn') => void;
 
   constructor() {
     // No initialization needed for direct Playwright execution
   }
 
+  /**
+   * Set a logger callback for capturing execution logs
+   */
+  setLogger(logger: (message: string, level?: 'log' | 'error' | 'warn') => void): void {
+    this.logger = logger;
+  }
+
+  /**
+   * Log a message using the configured logger
+   */
+  private log(message: string, level: 'log' | 'error' | 'warn' = 'log'): void {
+    if (this.logger) {
+      this.logger(message, level);
+    }
+    // No console fallback - logs are routed to consumer
+  }
+
   /**
    * Initialize the service
    */
   async initialize(): Promise<void> {
     try {
-      console.log('Initializing Playwright service...');
+      this.log('Initializing Playwright service...');
       
       // No specific initialization needed for direct Playwright execution
       this.isConnected = true;
-      console.log('Playwright service initialized successfully');
+      this.log('Playwright service initialized successfully');
     } catch (error) {
       throw new Error(`Failed to initialize Playwright service: ${error}`);
     }
@@ -109,7 +127,7 @@ export class PlaywrightMCPService {
 
     try {
       // Use the centralized browser initialization utility
-      const browserInstance = await initializeBrowser(config);
+      const browserInstance = await initializeBrowser(config, undefined, undefined, this.logger);
       browser = browserInstance.browser;
       context = browserInstance.context;
       page = browserInstance.page;
@@ -191,9 +209,9 @@ export class PlaywrightMCPService {
     try {
       // No cleanup needed for direct Playwright execution
       this.isConnected = false;
-      console.log('Playwright service closed');
+      this.log('Playwright service closed');
     } catch (error) {
-      console.error('Error during shutdown:', error);
+      this.log(`Error during shutdown: ${error}`, 'error');
     }
   }
 

package/src/progress-reporter.ts

@@ -0,0 +1,109 @@
+/**
+ * Progress Reporter Interface
+ * Allows consumers to track execution progress (logs, DB writes, etc.)
+ */
+
+/**
+ * Execution status for a step
+ * Matches scriptservice StepExecutionStatus enum
+ */
+export enum StepExecutionStatus {
+  SUCCESS = 'SUCCESS_STEP_EXECUTION',
+  FAILURE = 'FAILURE_STEP_EXECUTION',
+  IN_PROGRESS = 'IN_PROGRESS_STEP_EXECUTION'
+}
+
+/**
+ * Step progress report - camelCase for TypeScript
+ * Structure matches scriptservice's SmartTestExecutionStep / ScriptGenStep
+ */
+export interface StepProgress {
+  jobId: string;                      // Job ID for DB keying
+  stepId?: string;                    // Unique step identifier
+  stepNumber: number;                 // Step index (1-based)
+  description: string;                // Step description
+  code?: string;                      // Playwright command executed
+  screenshotDataUrl?: string;         // Screenshot as data URL (data:image/png;base64,...)
+  status: StepExecutionStatus;        // Execution status
+  error?: string;                     // Error message if failed
+  wasRepaired?: boolean;              // For repair mode
+  subActionCount?: number;            // Internal tracking
+  attempt?: number;                   // Internal tracking
+  
+  // Agent transparency (for orchestrator mode)
+  agentIteration?: number;            // Which iteration of the agent loop
+  agentReasoning?: string;            // Agent's reasoning for this iteration
+  agentSelfReflection?: any;          // SelfReflection guidance for next iteration
+  agentExperiences?: string[];        // Learnings from this iteration
+  agentToolsUsed?: string[];          // Tools the agent used
+  agentStatus?: string;               // Agent's status decision
+}
+
+/**
+ * Job progress report - camelCase for TypeScript
+ */
+export interface JobProgress {
+  jobId: string;                      // Job identifier
+  status: 'started' | 'in_progress' | 'completed' | 'failed';
+  currentStep?: number;               // Current step being executed
+  totalSteps?: number;                // Total steps in scenario
+  testName?: string;                  // Test name
+  script?: string;                    // Generated/repaired script
+  error?: string;                     // Error message if failed
+}
+
+/**
+ * Token usage tracking
+ */
+export interface TokenUsage {
+  jobId: string;
+  stepNumber?: number;
+  iteration?: number;
+  inputTokens: number;
+  outputTokens: number;
+  includesImage: boolean;
+  model: string;
+  timestamp: number;
+}
+
+/**
+ * Progress reporter interface for external consumers
+ */
+export interface ProgressReporter {
+  /**
+   * Called when a step starts, updates, or completes
+   * - VS Extension/GitHub: Log to console
+   * - Script Service: Write to DB, upload screenshot to GCS
+   */
+  onStepProgress?(progress: StepProgress): Promise<void>;
+  
+  /**
+   * Called when overall job status changes
+   * - VS Extension/GitHub: Log status
+   * - Script Service: Update job in DB
+   */
+  onJobProgress?(progress: JobProgress): Promise<void>;
+  
+  /**
+   * Called when script is generated/updated
+   */
+  onScriptGenerated?(jobId: string, script: string, testName: string): Promise<void>;
+  
+  /**
+   * Called when script is repaired/updated
+   */
+  onScriptRepaired?(jobId: string, originalScript: string, repairedScript: string, confidence: number): Promise<void>;
+  
+  /**
+   * Called when tokens are used (for cost tracking)
+   * - VS Extension/GitHub: Log token usage
+   * - Script Service: Store in DB for analytics
+   */
+  onTokensUsed?(usage: TokenUsage): Promise<void>;
+  
+  /**
+   * Generic logging (for environments that don't need structured progress)
+   */
+  log?(message: string, level?: 'log' | 'error' | 'warn'): void;
+}
+