testchimp-runner-core 0.0.32 → 0.0.34

package/plandocs/WHATS_NEW_v0.0.33.md

@@ -0,0 +1,183 @@
+# What's New in Runner-Core v0.0.33
+
+## Phase 1: Tactical Improvements - COMPLETE ✅
+
+---
+
+## 1. 📝 Note to Future Self (Cross-Step Memory)
+
+**The agent can now leave notes that persist across the entire scenario journey.**
+
+### How it works:
+```typescript
+// Step 1 - Login
+Agent: "Cookie modal appears after 2s. Dismiss it before interacting."
+→ Stored in memory.latestNote
+
+// Step 2 - Navigate to Dashboard
+Agent reads note from Step 1
+Agent: "Waiting 2s for cookie modal..."
+→ Dismisses modal proactively
+```
+
+### Scope:
+- ✅ Across iterations (within same step)
+- ✅ Across steps (entire scenario)
+- ✅ Free-form text (agent decides what's important)
+
+### Example notes:
+- **Tactical:** "Tried #menu, failed. Try SVG child next."
+- **Strategic:** "This app uses shadow DOM. Prefer CSS selectors over getByRole."
+- **Behavioral:** "Modals load after 2s delay. Wait before clicking."
+
+---
+
+## 2. 🎯 Percentage-Based Coordinate Fallback
+
+**When selectors fail, use visual positioning as last resort.**
+
+### Precision:
+- 3 decimal places (e.g., 15.755%, 8.500%)
+- ~1 pixel accuracy on most screens
+- Resolution-independent
+
+### Supported Actions:
+- **Click:** `{action: "click", xPercent: 15.755, yPercent: 8.500}`
+- **Fill:** `{action: "fill", xPercent: 30.000, yPercent: 25.000, value: "text"}`
+- **Drag:** `{action: "drag", xPercent: 10.000, yPercent: 50.000, toXPercent: 60.000, toYPercent: 50.000}`
+- **Hover, RightClick, DoubleClick, Scroll**
+
+### Auto-Activation:
+- Triggers after 3 consecutive selector failures
+- Limited to 2 coordinate attempts
+- Then gives up (stuck)
+
+---
+
+## 3. ⚡ Optimized Iteration Budget
+
+**Maximum 5 iterations per step** (down from 8)
+
+```
+Iterations 1-3: Playwright selectors (3 attempts)
+    with note-to-self between each
+    
+Iterations 4-5: Coordinates (2 attempts max)
+    If both fail → stuck
+```
+
+**Why:** Coordinates either work or don't - no point retrying 5+ times.
+
+---
+
+## 4. 🕐 Smart Timeout Handling (Earlier Fix)
+
+**Navigation operations now have appropriate timeouts:**
+- `waitForLoadState()`: 30 seconds (was 5s)
+- `goto()`: 30 seconds
+- Element operations: 5 seconds (unchanged)
+
+**Detects automatically:** Code scans command for navigation keywords.
+
+---
+
+## How Notes Work Across Steps
+
+### Example Scenario:
+
+```
+Step 1: Login
+  Iteration 1: Fill username → Success
+  Iteration 2: Fill password → Success
+  Iteration 3: Click login → Success
+  Agent note: "Login redirects to dashboard. Cookie modal appears after 2s."
+  
+Step 2: Navigate to Settings
+  Reads note from Step 1: "Cookie modal appears after 2s"
+  Iteration 1: 
+    - Wait 2s
+    - Dismiss modal
+    - Click Settings
+    → Success in 1 iteration! (note prevented wasted attempts)
+```
+
+**Benefit:** Agent builds up knowledge about the application and uses it in future steps.
+
+---
+
+## Comparison: Before vs After
+
+| Aspect | Before (v0.0.32) | After (v0.0.33) |
+|--------|------------------|------------------|
+| Iteration memory | None | Note to self (cross-step) |
+| Selector fails | Give up or loop | Coordinate fallback |
+| Max iterations | 8 per step | 5 per step |
+| Timeout handling | 5s for all | 30s for navigation |
+| Coordinate support | None | Full (click, fill, drag, etc.) |
+| Average iterations | ~4 per step | ~2.5 per step (estimated) |
+
+---
+
+## Testing Recommendations
+
+### Test 1: Note Continuity
+Create a scenario with repeated patterns:
+```
+- Login
+- Go to page A → encounter modal
+- Go to page B → should handle modal proactively
+```
+
+**Expected:** Step 2 learns from Step 1's note.
+
+### Test 2: Coordinate Fallback  
+Run PeopleHR scenario:
+```
+- Click hamburger menu (SVG icon)
+```
+
+**Expected:**
+- Iterations 1-3: Try selectors (may fail)
+- Iteration 4: Coordinates → succeeds
+- Generated script: `await page.mouse.click(x, y);`
+
+### Test 3: Timeout Fix
+Any scenario with:
+```
+- await page.waitForLoadState('networkidle');
+```
+
+**Expected:** No more 5s timeout errors.
+
+---
+
+## Migration
+
+**No code changes needed!** Existing code works as-is with improvements.
+
+**New response fields** (optional):
+- `noteToFutureSelf`: string (agent can optionally include)
+- `coordinateAction`: object (only when coordinate mode active)
+
+---
+
+## What's Next: Phase 2
+
+Phase 2 will add numbered element system for even better reliability:
+- Iteration 1: Playwright selector (1 attempt)
+- Iterations 2-3: Index commands CLICK[3] (2 attempts)
+- Iterations 4-5: Coordinates (2 attempts)
+
+**Target:** ~1.5 average iterations per step
+
+---
+
+## Status
+
+✅ **Built and Ready**  
+📦 **Version:** v0.0.33  
+🧪 **Status:** Ready for testing  
+📊 **Expected Impact:** 30-40% reduction in iterations  
+
+**Test now to validate improvements before Phase 2!**
+

package/src/llm-facade.ts

@@ -1,7 +1,7 @@
 import { PROMPTS } from './prompts';
 import { PageInfo } from './utils/page-info-utils';
 import { StepOperation } from './types';
-import { DEFAULT_MODEL, VISION_MODEL } from './model-constants';
+import { DEFAULT_MODEL, DEFAULT_SIMPLER_MODEL, VISION_MODEL } from './model-constants';
 import { LLMProvider, LLMRequest, LLMResponse } from './llm-provider';
 
 // LLM Response interfaces
@@ -140,7 +140,7 @@ export class LLMFacade {
   /**
    * Generate a test name from scenario description
    */
-  async generateTestName(scenario: string, model: string = DEFAULT_MODEL): Promise<string> {
+  async generateTestName(scenario: string, model: string = DEFAULT_SIMPLER_MODEL): Promise<string> {
     this.log('Generating test name with LLM...');
     
     const request: LLMRequest = {
@@ -163,7 +163,7 @@ export class LLMFacade {
   /**
    * Generate hashtags for semantic grouping
    */
-  async generateHashtags(scenario: string, model: string = 'gpt-4o-mini'): Promise<string[]> {
+  async generateHashtags(scenario: string, model: string = DEFAULT_SIMPLER_MODEL): Promise<string[]> {
     this.log('Generating hashtags with LLM...');
     
     const request: LLMRequest = {
@@ -280,7 +280,7 @@ Respond ONLY with valid JSON:
     errorMessage: string,
     attemptCount: number,
     pageInfo: any,
-    model: string = DEFAULT_MODEL
+    model: string = DEFAULT_SIMPLER_MODEL
   ): Promise<ScreenshotNeedResponse> {
     this.log('Assessing screenshot need with LLM...');
     
@@ -479,7 +479,7 @@ Respond ONLY with valid JSON:
   /**
    * Break down scenario into steps
    */
-  async breakdownScenario(scenario: string, model: string = DEFAULT_MODEL): Promise<ScenarioStep[]> {
+  async breakdownScenario(scenario: string, model: string = DEFAULT_SIMPLER_MODEL): Promise<ScenarioStep[]> {
     this.log('Breaking down scenario with LLM...');
     this.log(`📝 INPUT SCENARIO: ${scenario}`);
     
@@ -571,7 +571,7 @@ Respond ONLY with valid JSON:
   /**
    * Parse script into steps for AI repair
    */
-  async parseScriptIntoSteps(script: string, model: string = 'gpt-4o-mini'): Promise<Array<{ description: string; code: string; success?: boolean; error?: string }>> {
+  async parseScriptIntoSteps(script: string, model: string = DEFAULT_SIMPLER_MODEL): Promise<Array<{ description: string; code: string; success?: boolean; error?: string }>> {
     this.log('Parsing script into steps with LLM...');
     
     const request: LLMRequest = {
@@ -657,7 +657,7 @@ Respond ONLY with valid JSON:
   async assessRepairConfidence(
     originalScript: string,
     updatedScript: string,
-    model: string = DEFAULT_MODEL
+    model: string = DEFAULT_SIMPLER_MODEL
   ): Promise<RepairConfidenceResponse> {
     const request: LLMRequest = {
       model,
@@ -676,7 +676,7 @@ Respond ONLY with valid JSON:
     originalScript: string,
     updatedScript: string,
     newRepairAdvice: string,
-    model: string = 'gpt-4o-mini'
+    model: string = DEFAULT_SIMPLER_MODEL
   ): Promise<string> {
     const request: LLMRequest = {
       model,

package/src/llm-provider.ts

@@ -3,6 +3,15 @@
  * Abstraction for LLM calls - allows different implementations (backend proxy, local, etc.)
  */
 
+/**
+ * Labeled image for multi-image LLM requests
+ * Used for before/after comparisons or multi-screenshot analysis
+ */
+export interface LabeledImage {
+  label: string;      // e.g., "Before", "After", "Screenshot 1"
+  dataUrl: string;    // Base64 data URL (data:image/png;base64,...)
+}
+
 /**
  * LLM request - camelCase for TypeScript
  */
@@ -10,7 +19,8 @@ export interface LLMRequest {
   model?: string;
   systemPrompt?: string;
   userPrompt?: string;
-  imageUrl?: string; // For vision models (data URL)
+  imageUrl?: string;           // For vision models (data URL) - backward compatible, single image
+  images?: LabeledImage[];     // For multi-image requests (e.g., before/after comparison)
 }
 
 /**

package/src/model-constants.ts

@@ -5,14 +5,26 @@
  */
 
 /**
- * Default model for general LLM tasks:
- * - Scenario breakdown
- * - Command generation
- * - Goal completion checks
- * - Screenshot need assessment
+ * Default model for complex reasoning tasks:
+ * - Command generation (Playwright selectors)
+ * - Goal completion checks (critical decisions)
+ * - Repair suggestions (complex logic)
+ * - Agent orchestration (decision-making)
  */
 export const DEFAULT_MODEL = 'gpt-5-mini';
 
+/**
+ * Simpler model for structured/guided tasks:
+ * - Scenario breakdown (structured splitting)
+ * - Screenshot need assessment (yes/no decision)
+ * - Repair confidence assessment (comparison task)
+ * - Test name generation (text formatting)
+ * - Hashtag generation (semantic tags)
+ * - Script parsing (extract steps)
+ * - Final script merging (text concatenation)
+ */
+export const DEFAULT_SIMPLER_MODEL = 'gpt-4o-mini';
+
 /**
  * Vision-capable model for visual diagnostics:
  * - Screenshot analysis

package/src/orchestrator/index.ts

@@ -8,13 +8,14 @@ export { OrchestratorAgent } from './orchestrator-agent';
 // Tool system
 export { ToolRegistry, Tool, ToolParameter, ToolExecutionContext } from './tool-registry';
 
-// 5 information-gathering tools (state changes via Playwright commands)
+// 6 information-gathering tools (state changes via Playwright commands)
 export {
   TakeScreenshotTool,
   RecallHistoryTool,
   InspectPageTool,
   CheckPageReadyTool,
-  ExtractDataTool
+  ExtractDataTool,
+  VerifyActionResultTool
 } from './tools';
 
 // Types