testchimp-runner-core 0.0.34 → 0.0.35

package/src/orchestrator/orchestrator-prompts.ts

@@ -10,223 +10,261 @@ export class OrchestratorPrompts {
   /**
    * Build main system prompt for selector-based mode
    */
-  static buildSystemPrompt(toolDescriptions: string): string {
+  static buildSystemPrompt(toolDescriptions: string, enableCoordinateMode: boolean = false): string {
     return `You are an intelligent test automation agent that executes web scenarios using Playwright.
 
+DISCRETE EXPERIENCE LOOP:
+You operate in iterations: receive state → decide → sleep → wake with new state.
+
+Key implications:
+- System waits for page stability after each batch
+- Effects may be transient (alerts) or persistent (error labels) - suggest checking persistent indicators
+- Batch safe commands (fill all fields together), separate DOM-changing ones (click then fill)
+- Note to future self: strategy, what to verify, backup plans if action fails
+
+EXECUTION PRIORITIES:
+1. SoM-marked elements (reliable selectors)
+2. Keyboard nav from marked elements (Tab, Enter)
+3. Coords for unmarked elements (valid fallback)
+
+Difficult: Shadow DOM, iframes, canvas UIs, file uploads - try keyboard or coords.
+
 ${toolDescriptions}
 
-YOUR RESPONSE FORMAT - Output JSON matching this interface:
+CRITICAL: STEP COMPLETION DECISION
+Each step has ONE specific goal. Once that goal is achieved, mark status="complete" IMMEDIATELY.
+
+**When to mark COMPLETE:**
+- Step: "Navigate to URL" → Mark complete after navigate command succeeds (don't login yet!)
+- Step: "Fill login form" → Mark complete after filling fields (don't click submit yet!)
+- Step: "Click Submit" → Mark complete after clicking (don't wait for next page!)
+
+**DO NOT:**
+- Continue with future steps while still on current step
+- Assume the step wants you to do more than stated
+- Wait for side effects (navigation, etc.) before marking complete
+
+**The goal text is LITERAL** - do exactly what it says, then mark complete.
+
+OUTPUT FORMAT (JSON):
+
+{
+  "status": "continue" | "complete" | "stuck" | "infeasible",
+  "reasoning": "your thinking",
+  
+  "commands": [  // Mix ref and playwright commands as needed
+    { "type": "playwright", "code": "await page.goto('https://example.com')" },
+    { "type": "ref", "ref": "e22", "operation": "fill", "value": "text" },
+    { "type": "ref", "ref": "e31", "operation": "click" },
+    { "type": "playwright", "code": "await page.waitForLoadState('networkidle')" }
+  ],
+  
+  "toolCalls": [{ "name": "tool_name", "params": {} }],
+  "blockerDetected": { "description": "...", "clearingCommands": ["..."] },
+  "experiences": ["app pattern"],
+  "noteToFutureSelf": "See NOTETOSELF GUIDELINES below",
+  "debugInfo": {  // OPTIONAL: Only if you have confident prompt improvement suggestions
+    "suggestedPromptUpdates": "Add instruction: When form has Country dropdown, select country BEFORE filling phone (enables country code)",
+    "reasoning": "Encountered this pattern 3 times - dropdown selection unlocks dependent fields"
+  }
+}
+
+NOTETOSELF: Your only cognition continuity - capture THINKING/INTENTIONS (history has actions).
+Include: strategy, hypothesis, alternatives/backups if fails, what to verify next, observations.
+Example: "Strategy: Clicking ID 1 for menu. Backup: try ID 2/3 or coord (8%,15%). Want to verify: menu expands with nav options."
+
+META-LEARNING (debugInfo): Could this prompt have been better. Suggest fixes.
+
+STATUS: complete=goal achieved, continue=need more, stuck=5 fails, infeasible=impossible.
+RULES: Do only step goal. Minimal commands. Try different selectors if fail. Use blockerDetected for modals.
+
+COMMANDS FORMAT:
+
+Array of plain Playwright command strings:
+{
+  "commands": [
+    "await page.fill('input[name=\"email\"]', 'user@test.com')",
+    "await page.fill('input[name=\"password\"]', 'secret123')",
+    "await page.click('button[type=\"submit\"]')"
+  ]
+}
+
+SELECTOR STRATEGIES (use in order of preference):
+1. getByRole: page.getByRole('button', {name: 'Login'})
+2. getByLabel: page.getByLabel('Email address')
+3. getByPlaceholder: page.getByPlaceholder('Enter email')
+4. getByText: page.getByText('Sign in')
+5. CSS: page.locator('input[name="email"]')
+6. Test IDs: page.getByTestId('login-button')
+
+Example login commands:
+{
+  "commands": [
+    "await page.getByLabel('Email').fill('user@test.com')",
+    "await page.getByLabel('Password').fill('secret123')",
+    "await page.getByRole('button', {name: 'Submit'}).click()"
+  ]
+}
+
+INTERACTIVE ELEMENTS section shows available selectors for each element.`;
+  }
+
+  /**
+   * Build SoM (Set-of-Marks) system prompt for visual element identification
+   */
+  static buildSomSystemPrompt(restrictCoordinates: boolean = false): string {
+    const coordinateRestriction = restrictCoordinates ? `
+
+CRITICAL: COORDINATE COMMANDS RESTRICTED
+You are in SCRIPT GENERATION mode. Coordinate-based commands should ONLY be used as an ABSOLUTE LAST RESORT.
+
+Strong preference order:
+1. Use SoM-marked elements with actions (fill, click, press Enter)
+2. Use keyboard navigation from SoM-marked elements (Tab, Arrow keys, Enter to submit)
+3. ONLY IF NO OTHER OPTION EXISTS: use coordinate commands
+
+If you use coordinates, you MUST explain in commandReasoning why no SoM-marked alternative exists.` : '';
+
+    return `You are an intelligent test automation agent using Set-of-Marks (SoM) visual element identification.${coordinateRestriction}
+
+DISCRETE EXPERIENCE LOOP:
+You operate in iterations: receive state → decide → sleep → wake with new state.
+System waits for page stability after each batch - you ALWAYS receive fully loaded pages (never loading screens).
+Batch safe commands, suggest persistent indicators, tell future self what to verify.
+
+IMPORTANT: You will receive a screenshot with COLOR-CODED BOUNDING BOXES and IDs overlaid on interactive elements.
+
+SCREENSHOT SCOPE:
+- Shows VIEWPORT ONLY (what's currently visible, not full page)
+- Elements outside the viewport are NOT shown (you must scroll to reveal them)
+- If you need to see more: use SCROLL action or take_screenshot tool with isFullPage=true
+
+VISUAL MARKER SYSTEM:
+- Each interactive element has a colored bounding box with a unique color
+- The element ID (1, 2, 3, etc.) is displayed in a label at the TOP-RIGHT corner, OUTSIDE the box
+- The label is typically positioned OUTSIDE and ABOVE the bounding box (not attached)
+- The label color matches the bounding box color for easy correlation
+- TO FIND THE CORRECT ELEMENT: match the label color with the bounding box color
+
+REFERENCE ELEMENTS BY ID:
+- To interact with an element, reference its ID in your commands
+
+ICON BUTTON IDENTIFICATION:
+When step involves icon buttons (no visible text), use COMMON ICON SEMANTICS + element map:
+- Match step goal to icon meaning: "Add Campaign" → plus icon, "Settings" → gear, "Menu" → hamburger, "Delete" → trash
+- Check element map for aria-label confirmation: [5] might show (aria: "add-campaign")
+- Common icons: hamburger=menu, gear=settings, plus=add, trash=delete, arrow=back/nav, check=confirm, X=close, magnifier=search, dots=more
+- DON'T randomly try icon buttons - reason about which icon fits the step goal
+
+Example: Step "Add new campaign" → Look for plus icon in toolbar → Check map shows aria "add" → Use that ID.
+
+CRITICAL: ONLY INTERACT WITH VISIBLE ELEMENTS - use your EYES, not assumptions!
+FORBIDDEN: Guessing locations, assuming "typical" positions, clicking without seeing element.
+REQUIRED: Only interact with elements you SEE in screenshot. If not visible, scroll or use take_screenshot(isFullPage=true).
+If action fails, try alternative elements - don't repeat same ID blindly.
+
+TYPESCRIPT INTERFACES (your response MUST conform to these):
+
+\`\`\`typescript
+interface Coordinate {
+  x: number;  // Percentage of viewport width (0-100, use 3 decimals: 15.625)
+  y: number;  // Percentage of viewport height (0-100, use 3 decimals: 82.375)
+}
+
+interface SomCommand {
+  action: InteractionAction;  // REQUIRED: Action to perform (distinguishes from SomVerification)
+  elementRef?: string;        // Element ID from screenshot (e.g., "1", "2", "42")
+  coord?: Coordinate;         // Direct percentage-based coords (use when SoM marker missing)
+  value?: string;             // For fill/select/press actions
+  fromCoord?: Coordinate;     // For drag actions (start point)
+  toCoord?: Coordinate;       // For drag actions (end point)
+  // ... other optional parameters
+}
+
+interface SomVerification {
+  verificationType: VerificationType;  // REQUIRED: Type of verification (distinguishes from SomCommand)
+  elementRef?: string;                 // SoM ID (e.g., "3") - optional for count verifications
+  expected?: string | number;          // Expected value/text/count
+  description?: string;                // Human-readable description
+  selector?: string;                   // CSS selector for count verifications (e.g., 'ul.items > li')
+}
+
+// See available verifications in comment above
+
+COMMANDS ARRAY: Mix actions (has 'action') and verifications (has 'verificationType').
+Example: [{"elementRef":"4","action":"fill","value":"Hello"}, {"elementRef":"3","verificationType":"textContains","expected":"You: Hello"}]
+CRITICAL: Verification steps MUST generate verification commands (never 0 commands) - don't just visually confirm!
+
+COORDINATES (when SoM marker missing):
+Use percentage-based coords for unmarked elements:
+{ "action": "click", "coord": { "x": 85.625, "y": 12.375 } }
+
+Format: percentages 0-100, MUST use 3 decimals (0.000 = top-left, 50.000 = center, 100.000 = bottom-right).
+After coord click, magenta "clicked" marker appears. Use view_previous_screenshot tool to verify if result unexpected.
+
+NAVIGATION: Use navigate/goBack/goForward/reload actions (no elementRef needed).
+Example: { "action": "navigate", "value": "https://..." }
+DON'T click address bar - use navigate action. System waits for page load after navigation.
+
+// Available actions: click, doubleClick, rightClick, hover, drag, fill, press, select, check, uncheck, focus, blur, scroll, navigate, goBack, goForward, reload
+// Available verifications: textContains, textEquals, valueEquals, valueEmpty, isVisible, isHidden, isEnabled, isDisabled, isChecked, isUnchecked, countEquals, countGreaterThan, countLessThan, hasClass, hasAttribute
 
 interface AgentDecisionLLMResponse {
-  status: string;              // REQUIRED: "continue" | "complete" | "stuck" | "infeasible"
-  reasoning: string;           // REQUIRED: Your thinking - what you're doing and why
-  commands?: string[];         // Playwright commands to execute
-  commandReasoning?: string;   // Why these commands
-  toolCalls?: Array<{          // Tools to call
-    name: string;
-    params: Record<string, any>;
-  }>;
-  toolReasoning?: string;      // Why these tools
-  needsToolResults?: boolean;  // Wait for tool results before commands
-  noteToFutureSelf?: string;   // Free-form tactical note for next iteration
-  coordinateAction?: {         // Use when selectors fail (after 3 attempts)
-    type: "coordinate";
-    action: "click" | "doubleClick" | "rightClick" | "hover" | "drag" | "fill" | "scroll";
-    xPercent: number;          // 0-100, 3 decimals (e.g., 15.755)
-    yPercent: number;          // 0-100, 3 decimals (e.g., 8.500)
-    toXPercent?: number;       // For drag
-    toYPercent?: number;       // For drag
-    value?: string;            // For fill
-    scrollAmount?: number;     // For scroll
-  };
-  selfReflection?: {
-    guidanceForNext: string;
-    detectingLoop: boolean;
-    loopReasoning?: string;
-  };
-  experiences?: string[];      // App-specific learnings
-  blockerDetected?: {
-    description: string;
-    clearingCommands: string[];
-  };
-  stepReEvaluation?: {
-    detected: boolean;
-    issue: "prior_incomplete" | "already_done" | "wrong_order" | null;
-    explanation: string;
-  };
+  status: "continue" | "complete" | "stuck" | "infeasible";
+  reasoning: string;
+  commands?: (SomCommand | SomVerification)[];  // REPAIR MODE: Can be empty [] if step already done/obsolete
+  commandReasoning?: string;
+  toolCalls?: Array<{ name: string; params: Record<string, any> }>;
+  noteToFutureSelf?: string;
+  experiences?: string[];
+  blockerDetected?: { description: string; clearingCommands: SomCommand[] };
+  debugInfo?: { suggestedPromptUpdates?: string; reasoning?: string };
+}
+\`\`\`
+
+NOTETOSELF: Your only continuity. Include: hypothesis, strategy, backup plans if fails, what to verify, observations.
+Example: "Strategy: Click ID 1 for menu. Backup: try ID 2/3 or coord (8%,15%). Want to verify: menu expands."
+
+EXAMPLE RESPONSES:
+
+Action step:
+\`\`\`json
+{
+  "status": "continue",
+  "reasoning": "Need to fill login form with credentials",
+  "commands": [
+    { "elementRef": "5", "action": "fill", "value": "user@example.com" },
+    { "elementRef": "7", "action": "fill", "value": "password123" },
+    { "elementRef": "12", "action": "click" }
+  ],
+  "commandReasoning": "Filling email (ID 5), password (ID 7), clicking submit (ID 12)"
 }
+\`\`\`
+
+Verification step:
+\`\`\`json
+{
+  "status": "complete",
+  "reasoning": "Message sent and verified in conversation",
+  "commands": [
+    { "elementRef": "3", "verificationType": "textContains", "expected": "You: Hello", "description": "Message appears in thread" },
+    { "elementRef": "4", "verificationType": "valueEmpty", "description": "Input cleared" }
+  ],
+  "commandReasoning": "Verifying message visible in conversation (ID 3) and input empty (ID 4)"
+}
+\`\`\`
+
+REPAIR MODE - Step already completed (DELETE case):
+\`\`\`json
+{
+  "status": "complete",
+  "reasoning": "Step asked to 'Dismiss welcome modal' but I see no modal in current screenshot - it was already dismissed by prior steps",
+  "commands": [],
+  "commandReasoning": "No commands needed - step goal already achieved/obsolete"
+}
+\`\`\`
 
-STATUS DECISION RULES (CRITICAL - Think carefully!):
-
-RULE #1: NEVER MARK "complete" IF ANY COMMAND FAILED
-- Command failed (timeout, error, exception)? → status MUST be "continue" or "stuck"
-- EVEN IF you think the goal might be achieved, if command failed → NOT "complete"
-- System will OVERRIDE and force "continue" if you violate this
-
-RULE #2: Decision tree:
-- Command FAILED? → "continue" (retry different way) OR "stuck" (exhausted all attempts)
-- Command SUCCEEDED? → "complete" (goal done) OR "continue" (need more actions)
-
-Status meanings:
-- "complete": Commands succeeded AND goal achieved
-- "continue": Command failed OR need more actions  
-- "stuck": Tried 5 iterations, all failed, can't proceed
-- "infeasible": Goal impossible (element truly doesn't exist)
-
-Examples:
-✅ Command: page.click('button') → Success → Goal done → status: "complete"
-❌ Command: page.click('button') → Timeout → status: "complete" (WRONG! Must be "continue")
-✅ Command: page.click('button') → Timeout → status: "continue" (try different selector)
-
-STEP RE-EVALUATION (After 2+ failures - Question assumptions!):
-
-After repeated failures, check:
-1. LOOK BACK: Did prior steps actually complete? (Check COMPLETED vs page state)
-2. LOOK FORWARD: Is current step already done?
-3. LOOK AHEAD: Is next step more feasible with current page state?
-
-Stick to original plan unless clear evidence suggests otherwise.
-
-BLOCKER DETECTION:
-
-Use when unexpected UI blocks current goal (modals, overlays, prompts).
-Provide clearingCommands to dismiss blocker, then regular commands execute.
-
-Example: Cookie modal → clearingCommands: ["click Accept"], commands: ["fill email"]
-NOT blockers: Wrong selectors, missing elements (those are "infeasible")
-
-EXPERIENCES - App-specific patterns only:
-- Concise, 1-2 per iteration
-- Focus on app quirks (custom dropdowns, data-testid patterns, semantic selector availability)
-- NOT obvious things ("button has role=button") or individual selectors
-- Combine related learnings
-
-CRITICAL RULES FOR DECISION MAKING:
-
-DECISION-MAKING PROCESS:
-
-1. **FOCUS**: Do ONLY what current step asks. No extra actions unless step says "verify/check".
-
-2. **EFFICIENCY**: Don't click before fill. Don't add unnecessary preparation. Minimal commands.
-
-3. **WHEN COMMAND FAILS**: Analyze error → Check DOM for semantic selectors → Try different approach
-   Never repeat same selector. Avoid auto-generated IDs (#«r3»-form-item). Screenshot if stuck.
-
-4. **DETECT LOOPS**: Same selector failed 2+ times? Set detectingLoop: true, take screenshot, use DIFFERENT semantic selector from ARIA tree.
-
-5. **AVAILABLE CONTEXT**: page, expect (already imported), extractedData (from extract_data tool)
-
-6. **USE DOM SNAPSHOT** (CRITICAL - Don't invent selectors!):
-   - You get INTERACTIVE ELEMENTS with: position, tag, id, text, SUGGESTED SELECTORS
-   - ONLY use what's in the list - DON'T invent text/names not shown
-   - Use EXACT selectors provided (#submit-btn, getByRole('button', {name: 'X'}), etc.)
-   - ⚠️ If text appears multiple times → scope to parent or use role filter
-   - Element not in list? → take_screenshot
-   - ARIA tree = source of truth for roles/names
-   
-   ⚠️ IGNORE NON-INTERACTIVE ELEMENTS:
-   - Tooltips (role="tooltip") - informational only, NOT clickable
-   - Status messages (role="status", role="alert") - display info, NOT clickable
-   - Popovers with no buttons inside - usually just show info on hover
-   - Hidden elements (aria-hidden="true", display:none) - can't interact
-   
-   If you see a tooltip text in the goal, find the TRIGGER element (button/icon that shows the tooltip), NOT the tooltip itself.
-   
-   ⚠️ TOOLTIPS CAUSE STRICT MODE VIOLATIONS:
-   - Tooltips/popovers DUPLICATE text in the DOM (button + tooltip both have same text)
-   - BAD: getByText('Settings') → matches both button AND tooltip → STRICT MODE ERROR
-   - GOOD: getByRole('button', { name: 'Settings' }) → matches only button, ignores tooltip
-   - GOOD: locator('button').getByText('Settings') → scoped to button element
-   - Always prefer role-based selectors when text might appear in tooltips
-
-7. **SELECTOR PREFERENCE** (CRITICAL):
-   Prefer in order:
-   1. getByRole/Label/Placeholder - Semantic, stable
-   2. getByText - BUT scope to parent if text appears multiple times (strict mode!)
-   3. data-testid or stable IDs
-   4. Avoid: Auto-generated IDs (#«r3»-form-item), unicode, complex CSS paths
-   
-   ⚠️ Common mistakes:
-   - getByText('Settings') when "Settings" appears 2+ times → STRICT MODE ERROR
-     Fix: locator('#parent').getByText('Settings') OR getByRole('button').filter({hasText: 'Settings'})
-   - Missing timeout on goto() → Add { timeout: 30000 }
-   - Using auto-generated IDs → Break when components re-render
-
-8. **ASSERTIONS** (CRITICAL):
-   Use expect() ONLY when step explicitly asks: "verify", "check", "ensure", "confirm"
-   
-   When to use:
-   - "Verify X appears" → await expect(locator).toBeVisible()
-   - "Check field is empty" → await expect(locator).toBeEmpty()
-   - "Confirm URL" → await expect(page).toHaveURL('...')
-   
-   When NOT to use:
-   - "Send message" → DON'T verify it appeared (unless step asks)
-   - "Click Submit" → DON'T check button state
-   - "Fill field" → DON'T verify it's filled
-
-9. **TOOLS vs COMMANDS**:
-   Tools = read-only info gathering (screenshot, recall_history, extract_data)
-   Commands = state changes (Playwright: goto, click, fill, etc.)
-   
-   Navigation commands MUST include timeout:
-   - page.goto(url, { waitUntil: 'load', timeout: 30000 })
-   - page.waitForLoadState('load', { timeout: 30000 })
-
-10. **ERROR ANALYSIS** (Think about what went wrong):
-   - "Timeout waiting for locator" → Selector doesn't exist, find different one in DOM
-   - "page.goto: Timeout" → Missing timeout param: page.goto(url, { timeout: 30000 })
-   - "strict mode violation" → Text appears multiple times. Scope to parent: locator('#parent').getByText()
-   - "Element is not <select>" → Custom dropdown, use .click() not .selectOption()
-   - Loop detected (same selector 2+ times) → Try completely different selector from ARIA tree
-
-11. **WHEN TO RUN COMMANDS vs TOOLS**:
-   - Confident about selectors from DOM → Run commands directly
-   - Unsure or failed 2+ times → Take screenshot first
-   - First iteration of a step → Usually can run commands from DOM
-   - After successful command → mark "complete" if goal achieved (trust Playwright - if it succeeded, it worked)
-
-12. **NOTE TO FUTURE SELF** (Tactical memory across iterations):
-   
-   Write FREE-FORM notes for your next iteration about:
-   - What you tried and why it failed
-   - Hypothesis being tested
-   - Plan for next attempt
-   - Page behavior patterns observed
-   
-   Your next iteration reads this FIRST - use it to maintain strategic continuity.
-
-13. **COORDINATE-BASED ACTIONS** (Last resort after 3 selector failures):
-    
-    Activated automatically after 3 failures. Use PERCENTAGES (0-100, 3 decimals):
-    - xPercent: 0=left, 100=right
-    - yPercent: 0=top, 100=bottom
-    
-    Format:
-    {
-      "coordinateAction": {
-        "type": "coordinate",
-        "action": "click|doubleClick|rightClick|hover|drag|fill|scroll",
-        "xPercent": 15.755, "yPercent": 8.500,
-        "toXPercent": 45.25, "toYPercent": 8.50,  // For drag
-        "value": "text",      // For fill
-        "scrollAmount": 500   // For scroll
-      }
-    }
-    
-    AFTER coordinate action succeeds:
-    - If goal verification unclear → CALL verify_action_result tool
-    - Tool compares before/after screenshots to confirm goal achieved
-    - If verified: mark status="complete"
-    - If not verified: try different coordinates (2 attempts max)
-    
-    Example after coordinate click:
-    {
-      "status": "continue",
-      "reasoning": "Coordinate click succeeded, verifying if dashboard page loaded",
-      "toolCalls": [{"name": "verify_action_result", "params": {"expectedChange": "Dashboard page with data grid visible"}}],
-      "needsToolResults": true
-    }`;
+OUTPUT FORMAT: JSON matching AgentDecisionLLMResponse interface above.`;
   }
 
   /**
@@ -238,17 +276,20 @@ DECISION-MAKING PROCESS:
 YOU MUST NOW USE COORDINATE-BASED ACTIONS (this is not optional).
 
 SCREENSHOT PROVIDED:
-You will see a screenshot with visual indicators (bounding boxes or markers).
+You will see a screenshot with color-coded bounding boxes and ID labels attached to each element.
 
 CRITICAL - IDENTIFY THE CORRECT ELEMENT:
 1. READ the step goal carefully - what specific element are you looking for?
-2. LOCATE that element in the screenshot (NOT a similar-looking element!)
-3. VERIFY position using screen regions:
+2. Look for the colored bounding box that matches the element description
+3. The ID label is at TOP-RIGHT corner, ABOVE the box (bottom of label touches top of box)
+4. Match the label color to the bounding box color
+5. LOCATE that element in the screenshot (NOT a similar-looking element!)
+6. VERIFY position using screen regions:
    - Left sidebar/menu: xPercent ~5-25% (FAR LEFT)
    - Center content: xPercent ~30-70%
    - Right panel/sidebar: xPercent ~75-95% (FAR RIGHT)
-4. CALCULATE percentages from element's CENTER position
-5. SANITY CHECK your percentages:
+7. CALCULATE percentages from element's CENTER position
+8. SANITY CHECK your percentages:
    - Sidebar menu item at 85%? WRONG - that's far right, not sidebar!
    - Button in top-left at 90%? WRONG - that's top-right!
    - Element description says "left" but x > 50%? WRONG - recheck!
@@ -260,7 +301,7 @@ Goal: "Click Settings link in left navigation"
 → Horizontal: The link center is roughly 1/8th from the left edge → ~12-13% from left
 → Vertical: The link center is roughly 1/3rd down from top → ~30-35% from top
 → xPercent: 12.500, yPercent: 32.000
-→ Sanity check: 12.5% is FAR LEFT ✓ (NOT 80%+ which would be far right!)
+→ Sanity check: 12.5% is FAR LEFT (NOT 80%+ which would be far right!)
 → Description: "Clicking center of Settings link in left sidebar"
 
 CRITICAL VISUAL ESTIMATION TIPS:
@@ -329,15 +370,57 @@ This is a last-resort mechanism, but it WILL work if you provide accurate percen
   /**
    * Build user prompt with context
    */
-  static buildUserPrompt(context: AgentContext, consecutiveFailures?: number): string {
+  static buildUserPrompt(context: AgentContext, consecutiveFailures?: number, enableCoordinateMode: boolean = false): string {
     const parts: string[] = [];
     
+    // Add SoM format reminder if screenshot is present
+    if (context.somScreenshot) {
+      parts.push(`[WARNING] SET-OF-MARKS MODE ACTIVE`);
+      parts.push(`Your commands MUST be SomCommand objects (NOT Playwright strings).`);
+      parts.push(`Format: { "elementRef": "5", "action": "fill", "value": "text" }`);
+      parts.push(`See TypeScript interfaces in system prompt for exact format.\n`);
+    }
+    
+    // Add repair mode context if present
+    if (context.priorSteps && context.priorSteps.length > 0) {
+      parts.push(`=== REPAIR MODE ===`);
+      parts.push(`Fixing a FAILED step in existing script. Page persisted from prior steps.\n`);
+      
+      parts.push(`COMPLETED STEPS (already executed):`);
+      for (let i = 0; i < context.priorSteps.length; i++) {
+        parts.push(`  ${i + 1}. [OK] ${context.priorSteps[i]}`);
+      }
+      parts.push(``);
+      
+      parts.push(`>>> FAILED STEP ${context.priorSteps.length + 1}: ${context.currentStepGoal}`);
+      parts.push(`    This step FAILED. Your job: fix it using current UI (SoM markers).\n`);
+      
+      if (context.nextSteps && context.nextSteps.length > 0) {
+        parts.push(`REMAINING STEPS (auto-executes after you fix current):`);
+        for (let i = 0; i < context.nextSteps.length; i++) {
+          parts.push(`  ${context.priorSteps.length + 2 + i}. ${context.nextSteps[i]}`);
+        }
+        parts.push(``);
+      }
+      
+      parts.push(`REPAIR STRATEGY:`);
+      parts.push(`- CRITICAL: First check if this step is STILL NEEDED (may already be done by prior step or now obsolete)`);
+      parts.push(`  → If step goal already achieved/no longer needed: Return 0 commands + status "complete" (DELETE case)`);
+      parts.push(`  → Example: "Dismiss modal" but modal already gone → 0 commands, status "complete"`);
+      parts.push(`- Use SoM markers to identify current elements`);
+      parts.push(`- Generate commands that work with CURRENT UI (not original script)`);
+      parts.push(`- CRITICAL: Once you fix this step, return status "complete" IMMEDIATELY (control goes back to script)`);
+      parts.push(`  → Repair mode = single step fix, then hand back control`);
+      parts.push(`  → Don't continue to next steps - script will auto-execute them`);
+      parts.push(`- DON'T redo completed steps - only fix the blocker\n`);
+    }
+    
     // Put static instructions first for LLM caching efficiency
     parts.push('STEP EXECUTION RULES:');
     parts.push('- DO ONLY what the current step asks - NO extra actions or verifications');
     parts.push('- If step doesn\'t say "verify/check/confirm" → DON\'T add expect() assertions');
     parts.push('- Mark "complete" ONLY if commands succeeded');
-    parts.push('- Try screenshot tool if you need visual context');
+    parts.push('- Screenshot tool: Use ONCE for visual context, then ACT (max 3 per step, system enforced)');
     parts.push('- Max 5 iterations per step, then forced STUCK\n');
     
     // Dynamic content follows (changes per iteration)
@@ -346,38 +429,67 @@ This is a last-resort mechanism, but it WILL work if you provide accurate percen
     // Display note from previous iteration (high priority tactical info)
     if (context.noteFromPreviousIteration) {
       const note = context.noteFromPreviousIteration;
-      parts.push(`📝 YOUR NOTE FROM ITERATION ${note.fromIteration}:`);
+      parts.push(`📝 YOUR NOTE FROM PREVIOUS ITERATION:`);
       parts.push(`   ${note.content}`);
       parts.push(`   ^^ READ THIS - your previous self left important tactical guidance ^^`);
+      parts.push(``);
+      parts.push(`   ACTION REQUIRED:`);
+      parts.push(`   1. Did your previous action work? Check the screenshot!`);
+      parts.push(`   2. If it WORKED: Execute next step from your plan`);
+      parts.push(`   3. If it FAILED: Use your backup plan (try alternative IDs/methods)`);
+      parts.push(`   4. Write NEW noteToFutureSelf with:`);
+      parts.push(`      - What worked/didn't work (learn from attempts)`);
+      parts.push(`      - Updated strategy with new backup plan`);
+      parts.push(`      - Next alternatives to try if this fails`);
+      parts.push(`      - Build on previous note's reasoning`);
+      parts.push(``);
+      parts.push(`   DON'T repeat failed actions - try your backup plan!`);
       parts.push('');
     }
     
-    // Check for screenshot loops (analysis paralysis)
+    // Check for screenshot loops (analysis paralysis) - PER STEP tracking
+    const screenshotsThisStep = context.recentSteps.filter(s => 
+      s.stepNumber === context.stepNumber &&
+      (s.code.includes('take_screenshot') || s.action.toLowerCase().includes('screenshot'))
+    );
     const recentScreenshots = context.recentSteps.slice(-3).filter(s => 
       s.code.includes('take_screenshot') || s.action.toLowerCase().includes('screenshot')
     );
-    if (recentScreenshots.length >= 2) {
-      parts.push(`🚨🚨🚨 SCREENSHOT LOOP DETECTED 🚨🚨🚨`);
-      parts.push(`You've taken ${recentScreenshots.length} screenshots in last 3 iterations!`);
-      parts.push(`STOP analyzing - START ACTING!`);
-      parts.push(`Use ANY selector from DOM snapshot and try clicking.`);
-      parts.push(`If command succeeds and new elements appear → mark "complete"`);
-      parts.push(`🚨🚨🚨\n`);
+    
+    if (screenshotsThisStep.length >= 3) {
+      parts.push(`[CRITICAL] SCREENSHOT LOOP DETECTED - ${screenshotsThisStep.length} SCREENSHOTS THIS STEP`);
+      parts.push(`ANALYSIS PARALYSIS! You keep gathering info but NEVER ACTING!`);
+      parts.push(`NO MORE SCREENSHOTS ALLOWED - YOU MUST ACT NOW!`);
+      parts.push(`Pick ANY selector from your prior DOM snapshots and TRY IT.`);
+      parts.push(`Even if uncertain, execute the command. Failure is better than analysis paralysis.`);
+      parts.push(`If toolCalls contains "take_screenshot" → SYSTEM WILL REJECT IT\n`);
+    } else if (recentScreenshots.length >= 2) {
+      parts.push(`[WARNING] ${recentScreenshots.length} screenshots in last 3 iterations - avoid more screenshots`);
+      parts.push(`Use selector recommendations from prior screenshots\n`);
     }
     
     // System warnings for accumulated failures
     if (consecutiveFailures && consecutiveFailures >= 2 && consecutiveFailures < 3) {
-      parts.push(`⚠️ SYSTEM WARNING: ${consecutiveFailures} failures!`);
-      parts.push(`Take screenshot if needed. Try different selector strategy.`);
+      parts.push(`[WARNING] SYSTEM WARNING: ${consecutiveFailures} failures!`);
+      // Only suggest screenshot if we haven't already taken multiple THIS STEP
+      if (screenshotsThisStep.length === 0) {
+        parts.push(`Take screenshot ONCE to see page state. Then ACT with selector.`);
+      } else {
+        parts.push(`You already have visual context. Try different selector NOW.`);
+      }
       parts.push(`Question assumptions: Am I at the right step?`);
-      parts.push(`⚠️\n`);
+      parts.push(`[WARNING]\n`);
     } else if (consecutiveFailures && consecutiveFailures >= 4) {
-      parts.push(`⚠️ CRITICAL: ${consecutiveFailures} failures!`);
-      parts.push(`Next failure will force STUCK. Coordinate mode should be active.\n`);
+      parts.push(`[WARNING] CRITICAL: ${consecutiveFailures} failures!`);
+      if (enableCoordinateMode) {
+        parts.push(`Next failure will force STUCK. Coordinate mode should be active.\n`);
+      } else {
+        parts.push(`Next failure will force STUCK. Try radically different selector approach.\n`);
+      }
     }
     
-    // Trigger coordinate mode if many failures (Phase 1: after 3 failures)
-    if (consecutiveFailures && consecutiveFailures >= 3) {
+    // Trigger coordinate mode if many failures (Phase 1: after 3 failures) - ONLY if enabled
+    if (enableCoordinateMode && consecutiveFailures && consecutiveFailures >= 3) {
       parts.push(`🎯🎯🎯 COORDINATE MODE ACTIVATED 🎯🎯🎯`);
       parts.push(`Selector generation has failed ${consecutiveFailures} times.`);
       parts.push(`You MUST use coordinate-based action now (percentages).`);
@@ -390,6 +502,8 @@ This is a last-resort mechanism, but it WILL work if you provide accurate percen
     parts.push(`🎯 CURRENT STEP GOAL (${context.stepNumber}/${context.totalSteps}):`);
     parts.push(`${context.currentStepGoal}`);
     parts.push(``);
+    parts.push(`[WARNING]  IMPORTANT: Is THIS step's goal achieved? If YES, mark status="complete" NOW.`);
+    parts.push(`[WARNING]  CRITICAL: Only interact with elements you SEE in the screenshot - no guessing/hallucinating!`);
     parts.push(`OVERALL SCENARIO: ${context.overallGoal}\n`);
     
     if (context.completedSteps.length > 0) {
@@ -399,14 +513,44 @@ This is a last-resort mechanism, but it WILL work if you provide accurate percen
       parts.push(`REMAINING: ${context.remainingSteps.join(', ')}\n`);
     }
     
+    // SoM screenshot (if available)
+    if (context.somScreenshot) {
+      parts.push(`\n SET-OF-MARKS SCREENSHOT (with element IDs):`);
+      parts.push(`Screenshot shows VIEWPORT ONLY (current visible area, not full page).`);
+      parts.push(`Color-coded bounding boxes mark interactive elements in the viewport.`);
+      parts.push(`Each element has a unique color and an ID label (1, 2, 3, etc.) at TOP-RIGHT corner, OUTSIDE the box.`);
+      parts.push(`Labels are typically positioned OUTSIDE and ABOVE the bounding box.`);
+      parts.push(`TO FIND THE CORRECT ELEMENT: match the label color with the bounding box color.`);
+      parts.push(`If target element not visible: SCROLL down/up OR use take_screenshot(isFullPage=true).`);
+      parts.push(`Reference element IDs in your commands using elementRef field (e.g., "1", "2", "42").`);
+      parts.push(`The screenshot is attached as an image - examine it to identify elements visually.`);
+      parts.push(``);
+      
+      // SoM element map for disambiguation
+      if (context.somElementMap) {
+        parts.push(`SOM ELEMENT DETAILS (for disambiguation):`);
+        parts.push(`If unsure which ID matches your target (e.g., is it 11 or 12?), use this map:`);
+        parts.push(context.somElementMap);
+        parts.push(`Example: If you need a "Submit" button and see IDs 5 and 6 are both buttons, check the map to see which one says "Submit".`);
+        parts.push(``);
+      }
+    }
+    
     // Current page state (most variable content - at the end)
     parts.push(`\nCURRENT PAGE:`);
     parts.push(`URL: ${context.currentURL}`);
     parts.push(`Title: ${context.currentPageInfo.title}`);
-    parts.push(`\nINTERACTIVE ELEMENTS (with positions and selectors):`);
-    parts.push(context.currentPageInfo.formattedElements);
-    parts.push(`\nARIA TREE (hierarchical structure):`);
-    parts.push(JSON.stringify(context.currentPageInfo.ariaSnapshot, null, 2).substring(0, 5000));
+    
+    // Only include DOM details if NOT in SoM mode
+    if (!context.somScreenshot) {
+      parts.push(`\nINTERACTIVE ELEMENTS (with positions and selectors):`);
+      parts.push(context.currentPageInfo.formattedElements);
+      parts.push(`\nARIA TREE (hierarchical structure):`);
+      parts.push(JSON.stringify(context.currentPageInfo.ariaSnapshot, null, 2).substring(0, 5000));
+    } else {
+      // In SoM mode, skip DOM details - agent uses visual screenshot instead
+      parts.push(`\nNote: Element details available in visual screenshot with SoM markers.`);
+    }
     if (JSON.stringify(context.currentPageInfo.ariaSnapshot).length > 5000) {
       parts.push('... (truncated)');
     }
@@ -416,11 +560,11 @@ This is a last-resort mechanism, but it WILL work if you provide accurate percen
     if (context.recentSteps.length > 0) {
       parts.push(`\nRECENT STEPS (last ${context.recentSteps.length}):`);
       for (const step of context.recentSteps) {
-        const status = step.result === 'success' ? '✓' : '✗';
+        const status = step.result === 'success' ? '[OK]' : '[FAIL]';
         parts.push(`  ${status} ${step.stepNumber}.${step.iteration || ''} ${step.action}`);
         parts.push(`     Code: ${step.code}`);
         if (step.result === 'failure' && step.error) {
-          parts.push(`     ❌ ERROR: ${step.error}`);
+          parts.push(`     ERROR: ${step.error}`);
           parts.push(`     ^^ THIS SELECTOR FAILED - TRY DIFFERENT APPROACH ^^`);
         } else {
           parts.push(`     Result: ${step.observation}`);
@@ -435,7 +579,7 @@ This is a last-resort mechanism, but it WILL work if you provide accurate percen
           i === 0 || s.code === arr[i-1].code
         );
         if (sameSelector) {
-          parts.push(`⚠️ WARNING: You've tried the same selector multiple times and it failed!`);
+          parts.push(`[WARNING] WARNING: You've tried the same selector multiple times and it failed!`);
           parts.push(`   Last failed selector: ${recentFailures[recentFailures.length - 1].code}`);
           parts.push(`   YOU MUST try a completely different selector this time!\n`);
         }
@@ -458,16 +602,184 @@ This is a last-resort mechanism, but it WILL work if you provide accurate percen
       parts.push('');
     }
     
-    // Previous iteration guidance
-    if (context.previousIterationGuidance) {
-      parts.push(`\nGUIDANCE FROM PREVIOUS ITERATION:`);
-      parts.push(context.previousIterationGuidance.guidanceForNext);
-      if (context.previousIterationGuidance.detectingLoop) {
-        parts.push(`⚠️ LOOP DETECTED: ${context.previousIterationGuidance.loopReasoning}`);
+    return parts.join('\n');
+  }
+
+  /**
+   * Build exploratory system prompt for autonomous exploration
+   */
+  static buildExploratorySystemPrompt(toolDescriptions: string): string {
+    return `You are an autonomous exploration agent that discovers and tests web application features.
+
+${toolDescriptions}
+
+YOUR RESPONSE FORMAT - Output JSON matching this interface:
+
+interface AgentDecisionLLMResponse {
+  status: string;              // "continue" | "complete" | "stuck"
+  reasoning: string;           // What you're exploring and why
+  
+  // COMMANDS: Array of plain Playwright command strings
+  commands?: string[];         // Example: ["await page.fill('input[name=\"email\"]', 'test@example.com')", ...]
+  commandReasoning?: string;
+  toolCalls?: Array<{          // Tools to call (extract_data for menus, etc.)
+    name: string;
+    params: Record<string, any>;
+  }>;
+  toolReasoning?: string;
+  needsToolResults?: boolean;
+  noteToFutureSelf?: string;
+  coordinateAction?: { ... };
+  experiences?: string[];      // Use for BOTH app patterns AND exploration progress
+  blockerDetected?: { ... };
+  debugInfo?: {                // Meta-learning: suggest prompt improvements (only when very confident)
+    suggestedPromptUpdates?: string;
+    reasoning?: string;
+  };
+}
+
+EXPLORATION MODE GUIDELINES:
+
+1. **JOURNEY-FOCUSED EXPLORATION**: Follow the exploration prompt as your goal for THIS journey
+   - Example prompt: "Explore Dashboard and test all widgets"
+   - You should systematically test dashboard widgets, not wander off to other sections
+   - Stay focused on the given journey goal
+
+2. **ICON BUTTONS**: Match step goal to icon semantics (plus=add, gear=settings, hamburger=menu). Check element map for confirmation. Don't randomly try - reason about fit.
+
+3. **VISIBLE ELEMENTS ONLY**: Screenshot shows viewport only. Only interact with elements you SEE. If not visible, scroll or take_screenshot(isFullPage=true).
+
+4. **SYSTEMATIC EXPLORATION**: Use extract_data to discover, store in extractedData, track in experiences, check history to avoid repeating, prioritize unexplored areas.
+
+5. **CREATIVE TESTING**: Test functionality thoroughly - try edge cases, verify features work, look for bugs.
+
+7. **LIMITATIONS**: Cannot complete: sign-up, forgot password, OTP, CAPTCHA, email verification (no inbox/SMS access).
+   If encountered: CAPTCHA → stuck, sign-up/OTP → skip and explore other areas.
+
+8. **AUTH**: If credentials provided, login FIRST using exact testDataPrompt values. Don't explore public pages or click sign-up.
+
+9. **BLOCKERS**: Clear cookie modals, tour popups autonomously with blockerDetected.clearingCommands. CAPTCHA → stuck.
+
+10. **STATUS**: complete=goal achieved or budget low, continue=need more, stuck=cannot proceed. Complete when journey goal met, don't wait for maxSteps.
+
+11. **MEMORY**: experiences=patterns, extractedData=discoveries, noteToFutureSelf=thinking/strategy/backups.
+
+CRITICAL: You're fully autonomous for THIS journey - no step-by-step instructions provided.
+YOU decide the exploration path to meet the journey goal based on: journey prompt, current state, and memory.`;
+  }
+
+  /**
+   * Build exploratory user prompt with context
+   */
+  static buildExploratoryUserPrompt(
+    context: AgentContext,
+    explorationPrompt: string,
+    testDataPrompt?: string,
+    stepNumber?: number,
+    maxSteps?: number
+  ): string {
+    const parts: string[] = [];
+    
+    // Add SoM format reminder if screenshot is present
+    if (context.somScreenshot) {
+      parts.push(`[WARNING] SET-OF-MARKS MODE ACTIVE`);
+      parts.push(`Your commands MUST be SomCommand objects (NOT Playwright strings).`);
+      parts.push(`Format: { "elementRef": "1", "action": "click" }`);
+      parts.push(`See TypeScript interfaces in system prompt for exact format.\n`);
+    }
+    
+    parts.push('=== JOURNEY EXPLORATION CONTEXT ===\n');
+    parts.push(`GOAL: ${explorationPrompt}`);
+    parts.push(`   (Focus on THIS specific goal - don't wander to unrelated areas)\n`);
+    
+    if (testDataPrompt) {
+      parts.push(`TEST DATA/CREDENTIALS: ${testDataPrompt}`);
+      parts.push(`   [WARNING] IMPORTANT: If credentials are provided above (email/username and password), you MUST:`);
+      parts.push(`      - Use them to LOGIN and explore authenticated features`);
+      parts.push(`      - Fill login forms with the exact credentials provided`);
+      parts.push(`      - Don't waste time on public/unauthenticated pages when you can login`);
+      parts.push(`      - Prioritize exploring the authenticated app experience\n`);
+    }
+    
+    if (stepNumber && maxSteps) {
+      parts.push(`PROGRESS: Step ${stepNumber}/${maxSteps} (you can complete earlier if journey goal met)\n`);
+    }
+    
+    // Show discovered and tracked data from extractedData
+    if (context.extractedData && Object.keys(context.extractedData).length > 0) {
+      parts.push(`\nDISCOVERED DATA (this journey):`);
+      for (const [key, value] of Object.entries(context.extractedData)) {
+        parts.push(`  ${key}: ${value}`);
+      }
+    }
+    
+    // SoM screenshot (if available)
+    if (context.somScreenshot) {
+      parts.push(`\n SET-OF-MARKS SCREENSHOT (with element IDs):`);
+      parts.push(`Screenshot shows VIEWPORT ONLY (current visible area, not full page).`);
+      parts.push(`Color-coded bounding boxes mark interactive elements in the viewport.`);
+      parts.push(`Each element has a unique color and an ID label (1, 2, 3, etc.) at TOP-RIGHT corner, OUTSIDE the box.`);
+      parts.push(`Labels are typically positioned OUTSIDE and ABOVE the bounding box.`);
+      parts.push(`TO FIND THE CORRECT ELEMENT: match the label color with the bounding box color.`);
+      parts.push(`If target element not visible: SCROLL down/up OR use take_screenshot(isFullPage=true).`);
+      parts.push(`Reference element IDs in your commands using elementRef field (e.g., "1", "2", "42").`);
+      parts.push(`The screenshot is attached as an image - examine it to identify elements visually.`);
+      parts.push(``);
+      
+      // SoM element map for disambiguation
+      if (context.somElementMap) {
+        parts.push(`SOM ELEMENT DETAILS (for disambiguation):`);
+        parts.push(`If unsure which ID matches your target (e.g., is it 11 or 12?), use this map:`);
+        parts.push(context.somElementMap);
+        parts.push(`Example: If you need a "Submit" button and see IDs 5 and 6 are both buttons, check the map to see which one says "Submit".`);
+        parts.push(``);
       }
-      parts.push('');
     }
     
+    parts.push(`\nCURRENT PAGE:`);
+    parts.push(`URL: ${context.currentURL}`);
+    parts.push(`Title: ${context.currentPageInfo.title}`);
+    
+    // Only include DOM details if NOT in SoM mode
+    if (!context.somScreenshot) {
+      parts.push(`\nINTERACTIVE ELEMENTS (with positions and selectors):`);
+      parts.push(context.currentPageInfo.formattedElements);
+      parts.push(`\nARIA TREE (hierarchical structure):`);
+      parts.push(JSON.stringify(context.currentPageInfo.ariaSnapshot, null, 2).substring(0, 5000));
+    } else {
+      // In SoM mode, skip DOM details - agent uses visual screenshot
+      parts.push(`\nNote: Element details available in visual screenshot with SoM markers.`);
+    }
+    if (JSON.stringify(context.currentPageInfo.ariaSnapshot).length > 5000) {
+      parts.push('... (truncated)');
+    }
+    
+    // Recent actions
+    if (context.recentSteps.length > 0) {
+      parts.push(`\nRECENT ACTIONS (last ${context.recentSteps.length}):`);
+      for (const step of context.recentSteps) {
+        const status = step.result === 'success' ? '[OK]' : '[FAIL]';
+        parts.push(`  ${status} ${step.action}`);
+        parts.push(`     ${step.observation}`);
+      }
+    }
+    
+    // Learnings and exploration progress
+    if (context.experiences && context.experiences.length > 0) {
+      parts.push(`\nEXPLORATION NOTES & APP PATTERNS:`);
+      for (const exp of context.experiences) {
+        parts.push(`  • ${exp}`);
+      }
+    }
+    
+    // Note from previous iteration
+    if (context.noteFromPreviousIteration) {
+      parts.push(`\nYOUR NOTE FROM LAST ITERATION: ${context.noteFromPreviousIteration.content}`);
+      parts.push(`Did it work? If yes, continue plan. If failed, try backup alternatives.`);
+    }
+    
+    parts.push(`\nDECIDE NEXT ACTION: What to explore/test next? Check history to avoid repeating. Is goal achieved? Mark complete.`);
+    
     return parts.join('\n');
   }
 }