testchimp-runner-core 0.0.33 → 0.0.35

package/dist/orchestrator/page-som-handler.js

@@ -0,0 +1,1353 @@
+"use strict";
+/**
+ * PageSoMHandler - Set-of-Marks Handler for Visual Element Identification
+ * Manages SoM markers, canvas overlay, and semantic command execution
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PageSoMHandler = void 0;
+// @ts-nocheck - DOM APIs used in page.evaluate() browser context
+const test_1 = require("@playwright/test");
+const som_types_1 = require("./som-types");
+class PageSoMHandler {
+    constructor(page, logger) {
+        this.canvasInjected = false;
+        this.mutationsList = [];
+        this.mutationObserver = null; // MutationObserver only exists in browser context
+        this.page = page;
+        this.somMap = new Map();
+        this.logger = logger;
+    }
+    setPage(page) {
+        this.page = page;
+        this.somMap.clear();
+        this.canvasInjected = false;
+        this.disconnectMutationObserver();
+    }
+    async disconnectMutationObserver() {
+        if (this.mutationObserver || this.page) {
+            try {
+                // Clean up in page context
+                await this.page.evaluate(() => {
+                    if (window.__tcSomMutationObserver) {
+                        window.__tcSomMutationObserver.disconnect();
+                        delete window.__tcSomMutationObserver;
+                    }
+                    if (window.__tcSomMutations) {
+                        delete window.__tcSomMutations;
+                    }
+                });
+            }
+            catch (error) {
+                // Page may be closed or navigated away
+                this.logger?.(`[PageSoMHandler] Failed to disconnect mutation observer: ${error}`, 'warn');
+            }
+            this.mutationObserver = null;
+        }
+    }
+    /**
+     * Update SoM markers - extract interactive elements and draw overlay
+     * Note: Coordinate markers from previous iteration are NOT removed here
+     * They persist through screenshots so agent can see where coords landed
+     */
+    async updateSom() {
+        this.logger?.('[PageSoMHandler] Updating SoM markers...', 'log');
+        // Extract interactive elements and assign SoM IDs
+        const elements = await this.page.evaluate(() => {
+            const doc = document;
+            const elements = [];
+            let idCounter = 1;
+            // Query all interactive elements
+            const interactiveSelectors = [
+                'button', 'input', 'textarea', 'select', 'a[href]',
+                '[role="button"]', '[role="link"]', '[role="textbox"]',
+                '[role="checkbox"]', '[role="radio"]', '[role="combobox"]',
+                '[role="menu"]', '[role="menuitem"]', '[role="option"]',
+                '[aria-haspopup]', '[onclick]', '[type="submit"]',
+                '[role="tab"]', '[role="switch"]', '[role="spinbutton"]'
+            ];
+            const allInteractive = new Set();
+            interactiveSelectors.forEach(selector => {
+                doc.querySelectorAll(selector).forEach((el) => allInteractive.add(el));
+            });
+            // Also detect styled-as-interactive elements
+            doc.querySelectorAll('div, span, p, li, td').forEach((el) => {
+                const styles = window.getComputedStyle(el);
+                const hasClickHandler = el.onclick || el.getAttribute('onclick') ||
+                    el.hasAttribute('data-action') || el.hasAttribute('data-click');
+                if (styles.cursor === 'pointer' || hasClickHandler || el.tabIndex >= 0) {
+                    allInteractive.add(el);
+                }
+            });
+            // Remove elements that are descendants of "true" interactive elements
+            // (e.g., <span> inside <button> should not get separate marker)
+            // But keep elements inside generic containers (div, section, etc.)
+            const trueInteractiveTags = new Set([
+                'BUTTON', 'A', 'INPUT', 'TEXTAREA', 'SELECT', 'LABEL'
+            ]);
+            const topLevelInteractive = new Set();
+            allInteractive.forEach((el) => {
+                let hasInteractiveAncestor = false;
+                let parent = el.parentElement;
+                // Check if any ancestor is a "true" interactive element
+                while (parent) {
+                    if (allInteractive.has(parent) && trueInteractiveTags.has(parent.tagName)) {
+                        hasInteractiveAncestor = true;
+                        break;
+                    }
+                    parent = parent.parentElement;
+                }
+                if (!hasInteractiveAncestor) {
+                    topLevelInteractive.add(el);
+                }
+            });
+            // Filter to visible, non-occluded, enabled elements
+            topLevelInteractive.forEach((el) => {
+                const rect = el.getBoundingClientRect();
+                // Skip invisible elements (zero size)
+                if (rect.width === 0 || rect.height === 0)
+                    return;
+                // Skip hidden elements (display, visibility, opacity checks)
+                const styles = window.getComputedStyle(el);
+                if (styles.display === 'none' || styles.visibility === 'hidden' ||
+                    parseFloat(styles.opacity) === 0)
+                    return;
+                // Skip disabled elements (they can't be interacted with)
+                const isDisabled = el.disabled ||
+                    el.hasAttribute('disabled') ||
+                    el.getAttribute('aria-disabled') === 'true' ||
+                    el.getAttribute('data-disabled') === 'true' ||
+                    el.classList?.contains('disabled');
+                if (isDisabled)
+                    return;
+                // Z-index occlusion detection: Check if element is fully blocked
+                // Sample 5 points: center + 4 corners (slightly inset)
+                const centerX = rect.left + rect.width / 2;
+                const centerY = rect.top + rect.height / 2;
+                // Use smaller inset for small elements (min 1px, max 10% of dimension)
+                const inset = Math.max(1, Math.min(rect.width, rect.height) * 0.1);
+                const testPoints = [
+                    { x: centerX, y: centerY }, // center
+                    { x: rect.left + inset, y: rect.top + inset }, // top-left
+                    { x: rect.right - inset, y: rect.top + inset }, // top-right
+                    { x: rect.left + inset, y: rect.bottom - inset }, // bottom-left
+                    { x: rect.right - inset, y: rect.bottom - inset } // bottom-right
+                ];
+                // Element is visible if at least 1 test point hits it or a descendant
+                // (relaxed from 2 to handle small clickable icons better)
+                let visiblePoints = 0;
+                for (const point of testPoints) {
+                    const topEl = document.elementFromPoint(point.x, point.y);
+                    if (topEl && (topEl === el || el.contains(topEl) || topEl.contains(el))) {
+                        visiblePoints++;
+                    }
+                }
+                // Skip fully occluded elements (no visible points)
+                if (visiblePoints < 1) {
+                    return;
+                }
+                // Assign tc-som-id attribute
+                const somId = String(idCounter++);
+                el.setAttribute('tc-som-id', somId);
+                // Capture element details
+                const parent = el.parentElement;
+                elements.push({
+                    somId,
+                    tag: el.tagName.toLowerCase(),
+                    role: el.getAttribute('role') || el.tagName.toLowerCase(),
+                    text: el.textContent?.trim().substring(0, 50) || '',
+                    ariaLabel: el.getAttribute('aria-label') || '',
+                    placeholder: el.placeholder || '',
+                    name: el.getAttribute('name') || '',
+                    type: el.type || '',
+                    id: el.id || '',
+                    className: el.className || '',
+                    bbox: {
+                        x: Math.round(rect.x),
+                        y: Math.round(rect.y),
+                        width: Math.round(rect.width),
+                        height: Math.round(rect.height)
+                    },
+                    parent: parent ? {
+                        tag: parent.tagName.toLowerCase(),
+                        role: parent.getAttribute('role') || '',
+                        className: parent.className || '',
+                        text: parent.textContent?.trim().substring(0, 30) || ''
+                    } : undefined
+                });
+            });
+            return elements;
+        });
+        // Store in somMap
+        this.somMap.clear();
+        elements.forEach(el => this.somMap.set(el.somId, el));
+        // Inject/update canvas overlay with bounding boxes
+        await this.drawSomOverlay(elements);
+        this.logger?.(`[PageSoMHandler] Mapped ${elements.length} interactive elements`, 'log');
+        return elements.length;
+    }
+    /**
+     * Draw canvas overlay with bounding boxes and ID labels
+     */
+    async drawSomOverlay(elements) {
+        await this.page.evaluate((els) => {
+            const doc = document;
+            // Create or get canvas
+            let canvas = doc.getElementById('tc-som-canvas');
+            if (!canvas) {
+                canvas = doc.createElement('canvas');
+                canvas.id = 'tc-som-canvas';
+                canvas.style.cssText = `
+          position: fixed;
+          top: 0;
+          left: 0;
+          width: 100%;
+          height: 100%;
+          z-index: 2147483647;
+          pointer-events: none;
+          display: none;
+        `;
+                doc.body.appendChild(canvas);
+            }
+            // Set canvas dimensions to viewport size
+            canvas.width = window.innerWidth;
+            canvas.height = window.innerHeight;
+            const ctx = canvas.getContext('2d');
+            ctx.clearRect(0, 0, canvas.width, canvas.height);
+            // Make canvas visible for debugging (markers shown on live page)
+            canvas.style.display = 'block';
+            // 20 distinct colors with high contrast for readability
+            // Each color chosen to ensure the ID label is clearly visible
+            const COLOR_PALETTE = [
+                { stroke: '#CC0000', fill: '#FF4444', text: 'white' }, // Bright Red
+                { stroke: '#00AA00', fill: '#66FF66', text: 'black' }, // Bright Green
+                { stroke: '#0066CC', fill: '#3399FF', text: 'white' }, // Bright Blue
+                { stroke: '#FFAA00', fill: '#FFDD66', text: 'black' }, // Bright Orange
+                { stroke: '#CC00CC', fill: '#FF66FF', text: 'black' }, // Bright Magenta
+                { stroke: '#00AAAA', fill: '#66FFFF', text: 'black' }, // Bright Cyan
+                { stroke: '#FF6600', fill: '#FFAA66', text: 'black' }, // Orange-Red
+                { stroke: '#6600CC', fill: '#9966FF', text: 'white' }, // Purple
+                { stroke: '#00CC66', fill: '#66FFAA', text: 'black' }, // Sea Green
+                { stroke: '#FF0066', fill: '#FF66AA', text: 'white' }, // Hot Pink
+                { stroke: '#66CC00', fill: '#AAFF66', text: 'black' }, // Lime Green
+                { stroke: '#CC6600', fill: '#FFAA44', text: 'black' }, // Dark Orange
+                { stroke: '#0099FF', fill: '#66CCFF', text: 'black' }, // Sky Blue
+                { stroke: '#FF9999', fill: '#FFDDDD', text: 'black' }, // Light Coral
+                { stroke: '#AA5500', fill: '#FF8833', text: 'white' }, // Dark Orange/Brown
+                { stroke: '#5555AA', fill: '#8888FF', text: 'white' }, // Slate Blue
+                { stroke: '#AA0044', fill: '#FF4488', text: 'white' }, // Raspberry
+                { stroke: '#00AA88', fill: '#44FFCC', text: 'black' }, // Turquoise
+                { stroke: '#AA44AA', fill: '#DD88DD', text: 'black' }, // Orchid
+                { stroke: '#AAAA00', fill: '#FFFF66', text: 'black' } // Yellow-Green
+            ];
+            // Draw bounding boxes and labels
+            els.forEach((el, index) => {
+                const { bbox, somId } = el;
+                // Assign color based on element index (cycles through 20 colors)
+                const colors = COLOR_PALETTE[index % COLOR_PALETTE.length];
+                // Draw bounding box
+                ctx.strokeStyle = colors.stroke;
+                ctx.lineWidth = 3;
+                ctx.strokeRect(bbox.x, bbox.y, bbox.width, bbox.height);
+                // Draw ID label at TOP-RIGHT corner, OUTSIDE above the bounding box
+                // This prevents obscuring element content (text, icons, etc.)
+                const labelText = somId;
+                ctx.font = 'bold 14px Arial';
+                const textMetrics = ctx.measureText(labelText);
+                const textWidth = textMetrics.width;
+                const textHeight = 14;
+                const padding = 3;
+                const labelBoxWidth = textWidth + padding * 2;
+                const labelBoxHeight = textHeight + padding;
+                // Position label at top-right corner, ABOVE the bounding box
+                // Label is right-aligned with bbox, bottom of label touches top of bbox
+                const labelX = bbox.x + bbox.width - labelBoxWidth;
+                let labelY = bbox.y - labelBoxHeight;
+                // If no room above, place below the bounding box instead
+                if (labelY < 0) {
+                    labelY = bbox.y + bbox.height;
+                }
+                // Draw label background
+                ctx.fillStyle = colors.fill;
+                ctx.fillRect(labelX, labelY, labelBoxWidth, labelBoxHeight);
+                // Draw label text
+                ctx.fillStyle = colors.text;
+                ctx.textBaseline = 'top';
+                ctx.fillText(labelText, labelX + padding, labelY + padding / 2);
+                ctx.textBaseline = 'alphabetic'; // Reset
+            });
+        }, elements);
+        this.canvasInjected = true;
+    }
+    /**
+     * Get formatted SoM element map for agent context
+     * Returns concise, truncated element details to help disambiguate similar elements
+     */
+    getSomElementMap() {
+        if (this.somMap.size === 0) {
+            return 'No SoM elements mapped yet.';
+        }
+        const lines = [];
+        // Sort by somId (numeric)
+        const sortedEntries = Array.from(this.somMap.entries()).sort((a, b) => {
+            return parseInt(a[0]) - parseInt(b[0]);
+        });
+        for (const [somId, element] of sortedEntries) {
+            const parts = [];
+            // Tag (always present)
+            parts.push(element.tag);
+            // Text (truncate to 40 chars)
+            if (element.text && element.text.trim()) {
+                const truncated = element.text.trim().substring(0, 40);
+                parts.push(`"${truncated}${element.text.length > 40 ? '...' : ''}"`);
+            }
+            // Build attributes string
+            const attrs = [];
+            if (element.ariaLabel) {
+                attrs.push(`aria: "${element.ariaLabel.substring(0, 30)}"`);
+            }
+            if (element.placeholder) {
+                attrs.push(`placeholder: "${element.placeholder.substring(0, 30)}"`);
+            }
+            if (element.type && element.type !== 'text') {
+                attrs.push(`type: "${element.type}"`);
+            }
+            if (element.role && element.role !== 'generic') {
+                attrs.push(`role: "${element.role}"`);
+            }
+            if (element.name) {
+                attrs.push(`name: "${element.name.substring(0, 20)}"`);
+            }
+            // Combine: [1]: button "Submit" (aria: "submit-form", type: "submit")
+            const attrStr = attrs.length > 0 ? ` (${attrs.join(', ')})` : '';
+            lines.push(`[${somId}]: ${parts.join(' ')}${attrStr}`);
+        }
+        return lines.join('\n');
+    }
+    /**
+     * Get screenshot with optional SoM markers
+     */
+    async getScreenshot(includeSomMarkers, fullPage = false, quality = 60) {
+        // Show/hide canvas overlay
+        await this.page.evaluate((show) => {
+            const canvas = document.getElementById('tc-som-canvas');
+            if (canvas)
+                canvas.style.display = show ? 'block' : 'none';
+        }, includeSomMarkers);
+        // Capture screenshot
+        const buffer = await this.page.screenshot({
+            fullPage,
+            type: 'jpeg',
+            quality
+        });
+        // Keep markers visible for debugging - don't hide them
+        // Only hide if specifically requested (includeSomMarkers = false)
+        if (!includeSomMarkers) {
+            await this.page.evaluate(() => {
+                const canvas = document.getElementById('tc-som-canvas');
+                if (canvas)
+                    canvas.style.display = 'none';
+            });
+        }
+        return `data:image/jpeg;base64,${buffer.toString('base64')}`;
+    }
+    /**
+     * Run command with semantic selector generation and coordinate fallback
+     */
+    async runCommand(command, useSomIdBasedCommands = false) {
+        const failedAttempts = [];
+        const maxAttempts = 7; // Error budget per command
+        // Handle navigation actions (no element/coord required)
+        if (command.action === som_types_1.InteractionAction.NAVIGATE ||
+            command.action === som_types_1.InteractionAction.GO_BACK ||
+            command.action === som_types_1.InteractionAction.GO_FORWARD ||
+            command.action === som_types_1.InteractionAction.RELOAD) {
+            this.logger?.(`[PageSoMHandler] Executing navigation action: ${command.action}`, 'log');
+            try {
+                let playwrightCommand = '';
+                switch (command.action) {
+                    case som_types_1.InteractionAction.NAVIGATE:
+                        if (!command.value) {
+                            throw new Error('NAVIGATE action requires URL in value field');
+                        }
+                        await this.page.goto(command.value, { waitUntil: 'networkidle', timeout: 30000 });
+                        playwrightCommand = `await page.goto('${command.value}')`;
+                        break;
+                    case som_types_1.InteractionAction.GO_BACK:
+                        await this.page.goBack({ waitUntil: 'networkidle', timeout: 30000 });
+                        playwrightCommand = 'await page.goBack()';
+                        break;
+                    case som_types_1.InteractionAction.GO_FORWARD:
+                        await this.page.goForward({ waitUntil: 'networkidle', timeout: 30000 });
+                        playwrightCommand = 'await page.goForward()';
+                        break;
+                    case som_types_1.InteractionAction.RELOAD:
+                        await this.page.reload({ waitUntil: 'networkidle', timeout: 30000 });
+                        playwrightCommand = 'await page.reload()';
+                        break;
+                }
+                return {
+                    failedAttempts: [],
+                    successAttempt: {
+                        command: playwrightCommand,
+                        status: som_types_1.CommandRunStatus.SUCCESS
+                    },
+                    status: som_types_1.CommandRunStatus.SUCCESS
+                };
+            }
+            catch (error) {
+                return {
+                    failedAttempts: [{
+                            command: `Navigation action: ${command.action}`,
+                            status: som_types_1.CommandRunStatus.FAILURE,
+                            error: error.message
+                        }],
+                    error: error.message,
+                    status: som_types_1.CommandRunStatus.FAILURE
+                };
+            }
+        }
+        // Handle direct percentage-based coordinate commands
+        if (command.coord && !command.elementRef) {
+            this.logger?.(`[PageSoMHandler] Executing ${command.action} at percentage coords (${command.coord.x}%, ${command.coord.y}%)`, 'log');
+            try {
+                const result = await this.executePercentageCoordinateAction(command);
+                return {
+                    failedAttempts: [],
+                    successAttempt: {
+                        command: `Coordinate action: ${command.action} at (${command.coord.x}%, ${command.coord.y}%)`,
+                        status: som_types_1.CommandRunStatus.SUCCESS
+                    },
+                    status: som_types_1.CommandRunStatus.SUCCESS
+                };
+            }
+            catch (error) {
+                return {
+                    failedAttempts: [{
+                            command: `Coordinate action: ${command.action} at (${command.coord.x}%, ${command.coord.y}%)`,
+                            status: som_types_1.CommandRunStatus.FAILURE,
+                            error: error.message
+                        }],
+                    error: error.message,
+                    status: som_types_1.CommandRunStatus.FAILURE
+                };
+            }
+        }
+        // Lookup element in SoM map
+        if (!command.elementRef) {
+            return {
+                failedAttempts: [],
+                error: 'Command must have either elementRef or coord specified',
+                status: som_types_1.CommandRunStatus.FAILURE
+            };
+        }
+        const element = this.somMap.get(command.elementRef);
+        if (!element) {
+            return {
+                failedAttempts: [],
+                error: `Element with SoM ID "${command.elementRef}" not found in map`,
+                status: som_types_1.CommandRunStatus.FAILURE
+            };
+        }
+        this.logger?.(`[PageSoMHandler] Executing ${command.action} on element ${command.elementRef}`, 'log');
+        // Clear mutations list before command execution
+        this.mutationsList = [];
+        // Setup mutation observer for hover/focus actions
+        if (command.action === som_types_1.InteractionAction.HOVER || command.action === som_types_1.InteractionAction.FOCUS) {
+            await this.setupMutationObserver();
+        }
+        // Try semantic selectors first (unless useSomIdBasedCommands=true)
+        if (!useSomIdBasedCommands) {
+            const selectors = this.generateSemanticSelectors(element);
+            for (let i = 0; i < Math.min(selectors.length, maxAttempts); i++) {
+                const selector = selectors[i];
+                const result = await this.tryExecuteAction(selector, command, element);
+                if (result.status === som_types_1.CommandRunStatus.SUCCESS) {
+                    // Wait for mutations if hover/focus
+                    if (command.action === som_types_1.InteractionAction.HOVER || command.action === som_types_1.InteractionAction.FOCUS) {
+                        await this.page.waitForTimeout(500); // Wait for mutations to settle
+                        result.mutations = await this.filterRelevantMutations();
+                    }
+                    await this.disconnectMutationObserver();
+                    return result;
+                }
+                if (result.failedAttempts.length > 0) {
+                    failedAttempts.push(result.failedAttempts[0]);
+                }
+            }
+        }
+        else {
+            // Use tc-som-id attribute directly
+            const somIdSelector = {
+                type: 'locator',
+                value: `[tc-som-id="${command.elementRef}"]`
+            };
+            const result = await this.tryExecuteAction(somIdSelector, command, element);
+            if (result.status === som_types_1.CommandRunStatus.SUCCESS) {
+                await this.disconnectMutationObserver();
+                return result;
+            }
+            if (result.failedAttempts.length > 0) {
+                failedAttempts.push(result.failedAttempts[0]);
+            }
+        }
+        // Fallback to coordinates (more replayable)
+        this.logger?.(`[PageSoMHandler] Semantic selectors exhausted (${failedAttempts.length} attempts), falling back to coordinates`, 'warn');
+        const coordResult = await this.executeCoordinateAction(command, element);
+        await this.disconnectMutationObserver();
+        // Add failed attempts to coordinate result
+        if (coordResult.status === som_types_1.CommandRunStatus.SUCCESS) {
+            coordResult.failedAttempts = failedAttempts;
+        }
+        return coordResult;
+    }
+    /**
+     * Execute verification command and generate Playwright expect assertion
+     * Uses semantic selectors (reuses generateSemanticSelectors) for script portability
+     */
+    async executeVerification(verification) {
+        try {
+            let locatorStr = '';
+            let locator;
+            let element;
+            // Build locator using semantic selectors (like runCommand does)
+            if (verification.elementRef) {
+                element = this.somMap.get(verification.elementRef);
+                if (!element) {
+                    return {
+                        success: false,
+                        playwrightCommand: '',
+                        error: `Element with SoM ID ${verification.elementRef} not found`
+                    };
+                }
+                // Generate semantic selectors (reuse existing logic)
+                const semanticSelectors = this.generateSemanticSelectors(element);
+                if (semanticSelectors.length === 0) {
+                    return {
+                        success: false,
+                        playwrightCommand: '',
+                        error: `No semantic selectors available for element ${verification.elementRef}`
+                    };
+                }
+                // Try selectors in priority order to find working one
+                let workingSelector = null;
+                for (const typedSelector of semanticSelectors) {
+                    try {
+                        const testLocator = this.buildLocatorFromTypedSelector(typedSelector);
+                        await testLocator.first().waitFor({ state: 'attached', timeout: 1000 });
+                        workingSelector = typedSelector;
+                        break;
+                    }
+                    catch {
+                        continue;
+                    }
+                }
+                if (!workingSelector) {
+                    // Fall back to first selector even if not confirmed working
+                    workingSelector = semanticSelectors[0];
+                    this.logger?.(`[PageSoMHandler] No confirmed working selector, using first: ${this.formatSelector(workingSelector)}`, 'warn');
+                }
+                locator = this.buildLocatorFromTypedSelector(workingSelector);
+                locatorStr = this.formatSelector(workingSelector);
+            }
+            else if (verification.selector) {
+                // Direct CSS selector (for count verifications on non-SoM elements)
+                locatorStr = `page.locator('${verification.selector}')`;
+                locator = this.page.locator(verification.selector);
+            }
+            else {
+                return {
+                    success: false,
+                    playwrightCommand: '',
+                    error: 'Either elementRef or selector required for verification'
+                };
+            }
+            let expectCommand = '';
+            let verificationPassed = true;
+            // Generate Playwright command string FIRST (before executing, so we have it even if assertion fails)
+            switch (verification.verificationType) {
+                case som_types_1.VerificationType.TEXT_CONTAINS:
+                    expectCommand = `await expect(${locatorStr}).toContainText('${verification.expected}')`;
+                    break;
+                case som_types_1.VerificationType.TEXT_EQUALS:
+                    expectCommand = `await expect(${locatorStr}).toHaveText('${verification.expected}')`;
+                    break;
+                case som_types_1.VerificationType.VALUE_EQUALS:
+                    expectCommand = `await expect(${locatorStr}).toHaveValue('${verification.expected}')`;
+                    break;
+                case som_types_1.VerificationType.VALUE_EMPTY:
+                    expectCommand = `await expect(${locatorStr}).toHaveValue('')`;
+                    break;
+                case som_types_1.VerificationType.IS_VISIBLE:
+                    expectCommand = `await expect(${locatorStr}).toBeVisible()`;
+                    break;
+                case som_types_1.VerificationType.IS_HIDDEN:
+                    expectCommand = `await expect(${locatorStr}).toBeHidden()`;
+                    break;
+                case som_types_1.VerificationType.IS_ENABLED:
+                    expectCommand = `await expect(${locatorStr}).toBeEnabled()`;
+                    break;
+                case som_types_1.VerificationType.IS_DISABLED:
+                    expectCommand = `await expect(${locatorStr}).toBeDisabled()`;
+                    break;
+                case som_types_1.VerificationType.IS_CHECKED:
+                    expectCommand = `await expect(${locatorStr}).toBeChecked()`;
+                    break;
+                case som_types_1.VerificationType.IS_UNCHECKED:
+                    expectCommand = `await expect(${locatorStr}).not.toBeChecked()`;
+                    break;
+                case som_types_1.VerificationType.COUNT_EQUALS:
+                    expectCommand = `await expect(${locatorStr}).toHaveCount(${verification.expected})`;
+                    break;
+                case som_types_1.VerificationType.COUNT_GREATER_THAN:
+                    expectCommand = `expect(await ${locatorStr}.count()).toBeGreaterThan(${verification.expected})`;
+                    break;
+                case som_types_1.VerificationType.COUNT_LESS_THAN:
+                    expectCommand = `expect(await ${locatorStr}.count()).toBeLessThan(${verification.expected})`;
+                    break;
+                case som_types_1.VerificationType.HAS_CLASS:
+                    expectCommand = `await expect(${locatorStr}).toHaveClass(/${verification.expected}/)`;
+                    break;
+                case som_types_1.VerificationType.HAS_ATTRIBUTE:
+                    // Format: "attr:value" or just "attr" (use colon to avoid = in values)
+                    const parts = verification.expected.split(':', 2);
+                    const attr = parts[0];
+                    const value = parts[1];
+                    if (value) {
+                        expectCommand = `await expect(${locatorStr}).toHaveAttribute('${attr}', '${value}')`;
+                    }
+                    else {
+                        expectCommand = `await expect(${locatorStr}).toHaveAttribute('${attr}')`;
+                    }
+                    break;
+                default:
+                    return {
+                        success: false,
+                        playwrightCommand: '',
+                        error: `Unknown verification type: ${verification.verificationType}`
+                    };
+            }
+            // Now execute the assertion (may fail, but we already have the command string)
+            try {
+                switch (verification.verificationType) {
+                    case som_types_1.VerificationType.TEXT_CONTAINS:
+                        await (0, test_1.expect)(locator).toContainText(verification.expected, { timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.TEXT_EQUALS:
+                        await (0, test_1.expect)(locator).toHaveText(verification.expected, { timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.VALUE_EQUALS:
+                        await (0, test_1.expect)(locator).toHaveValue(verification.expected, { timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.VALUE_EMPTY:
+                        await (0, test_1.expect)(locator).toHaveValue('', { timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.IS_VISIBLE:
+                        await (0, test_1.expect)(locator).toBeVisible({ timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.IS_HIDDEN:
+                        await (0, test_1.expect)(locator).toBeHidden({ timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.IS_ENABLED:
+                        await (0, test_1.expect)(locator).toBeEnabled({ timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.IS_DISABLED:
+                        await (0, test_1.expect)(locator).toBeDisabled({ timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.IS_CHECKED:
+                        await (0, test_1.expect)(locator).toBeChecked({ timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.IS_UNCHECKED:
+                        await (0, test_1.expect)(locator).not.toBeChecked({ timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.COUNT_EQUALS:
+                        await (0, test_1.expect)(locator).toHaveCount(verification.expected, { timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.COUNT_GREATER_THAN:
+                        const countGT = await locator.count();
+                        (0, test_1.expect)(countGT).toBeGreaterThan(verification.expected);
+                        break;
+                    case som_types_1.VerificationType.COUNT_LESS_THAN:
+                        const countLT = await locator.count();
+                        (0, test_1.expect)(countLT).toBeLessThan(verification.expected);
+                        break;
+                    case som_types_1.VerificationType.HAS_CLASS:
+                        await (0, test_1.expect)(locator).toHaveClass(new RegExp(verification.expected), { timeout: 5000 });
+                        break;
+                    case som_types_1.VerificationType.HAS_ATTRIBUTE:
+                        const parts = verification.expected.split(':', 2);
+                        const attr = parts[0];
+                        const value = parts[1];
+                        if (value) {
+                            await (0, test_1.expect)(locator).toHaveAttribute(attr, value, { timeout: 5000 });
+                        }
+                        else {
+                            await (0, test_1.expect)(locator).toHaveAttribute(attr, { timeout: 5000 });
+                        }
+                        break;
+                }
+            }
+            catch (assertionError) {
+                // Assertion failed but we still want the command in the script
+                verificationPassed = false;
+                this.logger?.(`[PageSoMHandler] Verification assertion failed (non-fatal): ${assertionError.message}`, 'warn');
+            }
+            // Always return the command (even if assertion failed) - scripts need the expect
+            return {
+                success: verificationPassed,
+                playwrightCommand: expectCommand,
+                error: verificationPassed ? undefined : 'Assertion failed during generation'
+            };
+        }
+        catch (error) {
+            // Could not build command at all (selector not found, etc.)
+            return {
+                success: false,
+                playwrightCommand: '',
+                error: error.message
+            };
+        }
+    }
+    /**
+     * Generate typed semantic selectors from element details (no string parsing needed)
+     */
+    generateSemanticSelectors(element) {
+        const selectors = [];
+        // Priority 1: getByTestId (would need to capture in updateSom)
+        // Skipped for now - can add data-testid capture later
+        // Priority 2: Stable ID
+        if (element.id && !element.id.match(/^(rc_|:r[0-9]+:|__)/) && !element.id.includes('«')) {
+            selectors.push({ type: 'id', value: element.id });
+        }
+        // Priority 3: getByLabel
+        if (element.ariaLabel) {
+            selectors.push({ type: 'label', value: element.ariaLabel });
+        }
+        // Priority 4: getByRole
+        if (element.role && element.text) {
+            selectors.push({ type: 'role', value: element.role, roleOptions: { name: element.text } });
+        }
+        // Priority 5: getByPlaceholder
+        if (element.placeholder) {
+            selectors.push({ type: 'placeholder', value: element.placeholder });
+        }
+        // Priority 6: input[name]
+        if (element.name && ['input', 'textarea', 'select'].includes(element.tag)) {
+            selectors.push({ type: 'name', value: element.name });
+        }
+        // Priority 7: getByText
+        if (element.text) {
+            selectors.push({ type: 'text', value: element.text });
+        }
+        // Priority 8: Parent-scoped locator (generic Playwright selector)
+        if (element.parent?.className) {
+            const parentClass = element.parent.className.split(' ')[0];
+            if (parentClass) {
+                selectors.push({ type: 'locator', value: `.${parentClass} ${element.tag}` });
+            }
+        }
+        return selectors;
+    }
+    /**
+     * Try executing action with a typed selector (no string parsing)
+     */
+    async tryExecuteAction(typedSelector, command, element) {
+        // Format selector description for both success and error paths
+        const selectorDesc = this.formatSelector(typedSelector);
+        try {
+            // Build locator with optional parent chaining
+            const locator = this.buildLocatorFromTypedSelector(typedSelector);
+            // Execute action based on type
+            const playwrightCommand = await this.executeActionOnLocator(locator, command, selectorDesc);
+            return {
+                failedAttempts: [],
+                successAttempt: {
+                    command: playwrightCommand,
+                    status: som_types_1.CommandRunStatus.SUCCESS
+                },
+                status: som_types_1.CommandRunStatus.SUCCESS
+            };
+        }
+        catch (error) {
+            this.logger?.(`[PageSoMHandler] Selector "${selectorDesc}" failed: ${error.message}`, 'warn');
+            // Apply error-specific heuristics
+            return await this.applyHeuristicsAndRetry(typedSelector, command, element, error);
+        }
+    }
+    /**
+     * Apply intelligent heuristics based on error type and retry
+     */
+    async applyHeuristicsAndRetry(typedSelector, command, element, error) {
+        const errorMsg = error.message || String(error);
+        const selectorDesc = this.formatSelector(typedSelector);
+        // Heuristic 1: Strict mode violation → Try parent scoping
+        if (errorMsg.includes('strict mode violation')) {
+            if (typedSelector.type === 'locator' && element.parent?.className) {
+                this.logger?.(`[PageSoMHandler] Heuristic: Strict mode → trying parent scoping`, 'log');
+                return await this.tryRefinedSelector(typedSelector, command, element, error);
+            }
+            // Could add nth() selector here based on bbox position
+        }
+        // Heuristic 2: Timeout → Quick retry with scroll (once only, don't get stuck)
+        if (errorMsg.includes('Timeout') && !errorMsg.includes('not enabled')) {
+            this.logger?.(`[PageSoMHandler] Heuristic: Timeout → quick scroll + retry (once)`, 'log');
+            try {
+                const locator = this.buildLocatorFromTypedSelector(typedSelector);
+                // Try scroll (if element exists but not in viewport)
+                await locator.scrollIntoViewIfNeeded({ timeout: 1000 }).catch(() => { });
+                await this.page.waitForTimeout(500); // Reduced wait
+                const playwrightCommand = await this.executeActionOnLocator(locator, command, selectorDesc);
+                return {
+                    failedAttempts: [{
+                            command: selectorDesc,
+                            status: som_types_1.CommandRunStatus.FAILURE,
+                            error: 'Timeout (first attempt)'
+                        }],
+                    successAttempt: {
+                        command: `${playwrightCommand} (after scroll)`,
+                        status: som_types_1.CommandRunStatus.SUCCESS
+                    },
+                    status: som_types_1.CommandRunStatus.SUCCESS
+                };
+            }
+            catch (retryError) {
+                // Element likely doesn't exist or selector is wrong - move on to next selector
+                this.logger?.(`[PageSoMHandler] Scroll retry failed, moving to next selector`, 'log');
+            }
+        }
+        // Heuristic 3: Not enabled/not actionable → Try force flag
+        if (errorMsg.includes('not enabled') || errorMsg.includes('not editable') || errorMsg.includes('not actionable')) {
+            this.logger?.(`[PageSoMHandler] Heuristic: Not actionable → trying force flag`, 'log');
+            try {
+                const locator = this.buildLocatorFromTypedSelector(typedSelector);
+                const modifiedCommand = { ...command, force: true };
+                const playwrightCommand = await this.executeActionOnLocator(locator, modifiedCommand, selectorDesc);
+                return {
+                    failedAttempts: [{
+                            command: selectorDesc,
+                            status: som_types_1.CommandRunStatus.FAILURE,
+                            error: errorMsg
+                        }],
+                    successAttempt: {
+                        command: `${playwrightCommand} (with force)`,
+                        status: som_types_1.CommandRunStatus.SUCCESS
+                    },
+                    status: som_types_1.CommandRunStatus.SUCCESS
+                };
+            }
+            catch (forceError) {
+                // Force didn't work
+            }
+        }
+        // Heuristic 4: Element moving/detached → Wait for stability
+        if (errorMsg.includes('detached') || errorMsg.includes('moving')) {
+            this.logger?.(`[PageSoMHandler] Heuristic: Element unstable → waiting for stability`, 'log');
+            try {
+                await this.page.waitForLoadState('domcontentloaded');
+                await this.page.waitForTimeout(1000);
+                const locator = this.buildLocatorFromTypedSelector(typedSelector);
+                const playwrightCommand = await this.executeActionOnLocator(locator, command, selectorDesc);
+                return {
+                    failedAttempts: [{
+                            command: selectorDesc,
+                            status: som_types_1.CommandRunStatus.FAILURE,
+                            error: errorMsg
+                        }],
+                    successAttempt: {
+                        command: `${playwrightCommand} (after stability wait)`,
+                        status: som_types_1.CommandRunStatus.SUCCESS
+                    },
+                    status: som_types_1.CommandRunStatus.SUCCESS
+                };
+            }
+            catch (stabilityError) {
+                // Stability wait didn't help
+            }
+        }
+        // No heuristics worked - return failure
+        return {
+            failedAttempts: [{
+                    command: selectorDesc,
+                    status: som_types_1.CommandRunStatus.FAILURE,
+                    error: errorMsg
+                }],
+            status: som_types_1.CommandRunStatus.FAILURE
+        };
+    }
+    /**
+     * Build Playwright locator from typed selector (supports chaining)
+     */
+    buildLocatorFromTypedSelector(typedSelector) {
+        // Get base locator (page or parent)
+        const base = typedSelector.parent
+            ? this.buildLocatorFromTypedSelector(typedSelector.parent)
+            : this.page;
+        // Build locator from base
+        switch (typedSelector.type) {
+            case 'id':
+                return base.locator(`#${typedSelector.value}`);
+            case 'testId':
+                return base.getByTestId(typedSelector.value);
+            case 'label':
+                return base.getByLabel(typedSelector.value);
+            case 'role':
+                return base.getByRole(typedSelector.value, typedSelector.roleOptions);
+            case 'placeholder':
+                return base.getByPlaceholder(typedSelector.value);
+            case 'text':
+                return base.getByText(typedSelector.value);
+            case 'title':
+                return base.getByTitle(typedSelector.value);
+            case 'altText':
+                return base.getByAltText(typedSelector.value);
+            case 'name':
+                return base.locator(`[name="${typedSelector.value}"]`);
+            case 'locator':
+                // Generic locator - supports CSS, text=, has-text=, chaining, etc.
+                return base.locator(typedSelector.value);
+            default:
+                throw new Error(`Unknown selector type: ${typedSelector.type}`);
+        }
+    }
+    /**
+     * Format typed selector for logging (supports chaining)
+     */
+    formatSelector(sel) {
+        let formatted;
+        switch (sel.type) {
+            case 'id':
+                formatted = `#${sel.value}`;
+                break;
+            case 'testId':
+                formatted = `getByTestId('${sel.value}')`;
+                break;
+            case 'label':
+                formatted = `getByLabel('${sel.value}')`;
+                break;
+            case 'role':
+                formatted = `getByRole('${sel.value}', {name: '${sel.roleOptions?.name}'})`;
+                break;
+            case 'placeholder':
+                formatted = `getByPlaceholder('${sel.value}')`;
+                break;
+            case 'text':
+                formatted = `getByText('${sel.value}')`;
+                break;
+            case 'title':
+                formatted = `getByTitle('${sel.value}')`;
+                break;
+            case 'altText':
+                formatted = `getByAltText('${sel.value}')`;
+                break;
+            case 'name':
+                formatted = `[name="${sel.value}"]`;
+                break;
+            case 'locator':
+                formatted = sel.value;
+                break;
+            default: formatted = String(sel.value);
+        }
+        // Add parent chain if exists
+        if (sel.parent) {
+            return `${this.formatSelector(sel.parent)}.locator(${formatted})`;
+        }
+        return formatted;
+    }
+    /**
+     * Execute action on locator based on command type
+     */
+    async executeActionOnLocator(locator, command, selectorDesc) {
+        const { action, value, force, scrollAmount } = command;
+        const selector = selectorDesc || 'locator';
+        switch (action) {
+            case som_types_1.InteractionAction.CLICK:
+                await locator.click({ force });
+                return `await ${selector}.click()`;
+            case som_types_1.InteractionAction.DOUBLE_CLICK:
+                await locator.dblclick({ force });
+                return `await ${selector}.dblclick()`;
+            case som_types_1.InteractionAction.RIGHT_CLICK:
+                await locator.click({ button: 'right', force });
+                return `await ${selector}.click({ button: 'right' })`;
+            case som_types_1.InteractionAction.FILL:
+                await locator.fill(value || '', { force });
+                return `await ${selector}.fill('${value}')`;
+            case som_types_1.InteractionAction.TYPE:
+                await locator.pressSequentially(value || '', { delay: command.delay || 50 });
+                return `await ${selector}.pressSequentially('${value}')`;
+            case som_types_1.InteractionAction.CLEAR:
+                await locator.clear({ force });
+                return `await ${selector}.clear()`;
+            case som_types_1.InteractionAction.PRESS:
+                await locator.press(value || 'Enter');
+                return `await ${selector}.press('${value}')`;
+            case som_types_1.InteractionAction.SELECT:
+                await locator.selectOption(value || '');
+                return `await ${selector}.selectOption('${value}')`;
+            case som_types_1.InteractionAction.CHECK:
+                await locator.check({ force });
+                return `await ${selector}.check()`;
+            case som_types_1.InteractionAction.UNCHECK:
+                await locator.uncheck({ force });
+                return `await ${selector}.uncheck()`;
+            case som_types_1.InteractionAction.HOVER:
+                await locator.hover({ force });
+                return `await ${selector}.hover()`;
+            case som_types_1.InteractionAction.FOCUS:
+                await locator.focus();
+                return `await ${selector}.focus()`;
+            case som_types_1.InteractionAction.BLUR:
+                await locator.blur();
+                return `await ${selector}.blur()`;
+            case som_types_1.InteractionAction.SCROLL_INTO_VIEW:
+                await locator.scrollIntoViewIfNeeded();
+                return `await ${selector}.scrollIntoViewIfNeeded()`;
+            case som_types_1.InteractionAction.SCROLL:
+                if (scrollAmount) {
+                    await locator.evaluate((el, amount) => {
+                        el.scrollBy(0, amount);
+                    }, scrollAmount);
+                    return `await ${selector}.evaluate(el => el.scrollBy(0, ${scrollAmount}))`;
+                }
+                break;
+            case som_types_1.InteractionAction.DRAG:
+                if (command.toCoord) {
+                    await locator.dragTo(this.page.locator('body'), {
+                        targetPosition: { x: command.toCoord.x, y: command.toCoord.y }
+                    });
+                    return `await ${selector}.dragTo(target)`;
+                }
+                break;
+            default:
+                throw new Error(`Unsupported action: ${action}`);
+        }
+        return `await ${selector}.${action}()`;
+    }
+    /**
+     * Execute coordinate-based action as fallback
+     */
+    /**
+     * Draw a visual marker at coordinate position (for debugging in headed mode)
+     * Marker persists through screenshots so agent can see where coords landed
+     */
+    async drawCoordinateMarker(pixelX, pixelY) {
+        await this.page.evaluate(({ x, y }) => {
+            // Remove any existing coordinate marker and labels
+            const existing = document.getElementById('tc-coord-marker');
+            if (existing)
+                existing.remove();
+            const existingLabels = Array.from(document.querySelectorAll('div')).filter(el => el.textContent === 'clicked' && el.style.position === 'fixed');
+            existingLabels.forEach(label => label.remove());
+            // Create marker element
+            const marker = document.createElement('div');
+            marker.id = 'tc-coord-marker';
+            marker.style.cssText = `
+        position: fixed;
+        left: ${x}px;
+        top: ${y}px;
+        width: 24px;
+        height: 24px;
+        margin-left: -12px;
+        margin-top: -12px;
+        border-radius: 50%;
+        background: rgba(255, 0, 255, 0.8);
+        border: 3px solid rgba(255, 255, 0, 0.95);
+        z-index: 2147483647;
+        pointer-events: none;
+        box-shadow: 0 0 10px rgba(255, 0, 255, 0.6);
+        animation: tc-pulse 0.5s ease-out;
+      `;
+            // Add label showing this was a coord click
+            const label = document.createElement('div');
+            label.style.cssText = `
+        position: fixed;
+        left: ${x + 15}px;
+        top: ${y - 20}px;
+        background: rgba(255, 0, 255, 0.9);
+        color: white;
+        padding: 2px 6px;
+        border-radius: 3px;
+        font-size: 11px;
+        font-weight: bold;
+        font-family: Arial;
+        z-index: 2147483647;
+        pointer-events: none;
+        white-space: nowrap;
+      `;
+            label.textContent = 'clicked';
+            // Add pulsing animation
+            const style = document.createElement('style');
+            style.textContent = `
+        @keyframes tc-pulse {
+          0% { transform: scale(0); opacity: 0; }
+          50% { transform: scale(1.5); opacity: 1; }
+          100% { transform: scale(1); opacity: 1; }
+        }
+      `;
+            document.head.appendChild(style);
+            document.body.appendChild(marker);
+            document.body.appendChild(label);
+        }, { x: pixelX, y: pixelY });
+        // Brief pause for animation to complete, then marker stays visible
+        await this.page.waitForTimeout(300);
+    }
+    /**
+     * Remove coordinate marker from page (used when cleaning up or replacing)
+     */
+    async removeCoordinateMarker() {
+        await this.page.evaluate(() => {
+            // Remove marker circle
+            const marker = document.getElementById('tc-coord-marker');
+            if (marker)
+                marker.remove();
+            // Remove all "clicked" labels (in case there are multiple)
+            const labels = Array.from(document.querySelectorAll('div')).filter(el => el.textContent === 'clicked' && el.style.position === 'fixed' && el.style.background?.includes('255, 0, 255'));
+            labels.forEach(label => label.remove());
+        });
+    }
+    /**
+     * Execute action using percentage-based coordinates
+     */
+    async executePercentageCoordinateAction(command) {
+        if (!command.coord) {
+            throw new Error('Coordinate is required for percentage-based commands');
+        }
+        // Get viewport dimensions
+        const viewport = this.page.viewportSize();
+        if (!viewport) {
+            throw new Error('Could not determine viewport size');
+        }
+        // Convert percentage to pixels
+        const pixelX = Math.round((command.coord.x / 100) * viewport.width);
+        const pixelY = Math.round((command.coord.y / 100) * viewport.height);
+        this.logger?.(`[PageSoMHandler] Using percentage coords: (${command.coord.x}%, ${command.coord.y}%) -> pixels (${pixelX}, ${pixelY})`, 'log');
+        // Draw visual marker at target position (for headed mode debugging)
+        await this.drawCoordinateMarker(pixelX, pixelY);
+        const { action, value } = command;
+        let playwrightCommand = '';
+        try {
+            switch (action) {
+                case som_types_1.InteractionAction.CLICK:
+                case som_types_1.InteractionAction.DOUBLE_CLICK:
+                case som_types_1.InteractionAction.RIGHT_CLICK:
+                    const clickCount = action === som_types_1.InteractionAction.DOUBLE_CLICK ? 2 : 1;
+                    const button = action === som_types_1.InteractionAction.RIGHT_CLICK ? 'right' :
+                        command.button || 'left';
+                    await this.page.mouse.click(pixelX, pixelY, {
+                        button,
+                        clickCount,
+                        delay: command.delay
+                    });
+                    playwrightCommand = `await page.mouse.click(${pixelX}, ${pixelY}${button !== 'left' ? `, { button: '${button}' }` : ''})`;
+                    break;
+                case som_types_1.InteractionAction.FILL:
+                case som_types_1.InteractionAction.TYPE:
+                    // Click to focus, then type
+                    await this.page.mouse.click(pixelX, pixelY);
+                    await this.page.waitForTimeout(100); // Brief wait for focus
+                    if (action === som_types_1.InteractionAction.FILL) {
+                        await this.page.keyboard.type(value || '');
+                    }
+                    else {
+                        await this.page.keyboard.type(value || '', { delay: command.delay || 50 });
+                    }
+                    playwrightCommand = `await page.mouse.click(${pixelX}, ${pixelY}); await page.keyboard.type('${value}')`;
+                    break;
+                case som_types_1.InteractionAction.HOVER:
+                    await this.page.mouse.move(pixelX, pixelY);
+                    playwrightCommand = `await page.mouse.move(${pixelX}, ${pixelY})`;
+                    break;
+                case som_types_1.InteractionAction.DRAG:
+                    if (!command.toCoord) {
+                        throw new Error('toCoord is required for drag action');
+                    }
+                    const toPixelX = Math.round((command.toCoord.x / 100) * viewport.width);
+                    const toPixelY = Math.round((command.toCoord.y / 100) * viewport.height);
+                    await this.page.mouse.move(pixelX, pixelY);
+                    await this.page.mouse.down();
+                    await this.page.mouse.move(toPixelX, toPixelY);
+                    await this.page.mouse.up();
+                    playwrightCommand = `await page.mouse.move(${pixelX}, ${pixelY}); await page.mouse.down(); await page.mouse.move(${toPixelX}, ${toPixelY}); await page.mouse.up()`;
+                    break;
+                default:
+                    throw new Error(`Coordinate-based execution not supported for action: ${action}`);
+            }
+            return {
+                failedAttempts: [],
+                successAttempt: {
+                    command: playwrightCommand,
+                    status: som_types_1.CommandRunStatus.SUCCESS
+                },
+                status: som_types_1.CommandRunStatus.SUCCESS
+            };
+        }
+        catch (error) {
+            throw new Error(`Failed to execute percentage coordinate action: ${error.message}`);
+        }
+    }
+    async executeCoordinateAction(command, element) {
+        try {
+            // Calculate center of bounding box
+            const centerX = element.bbox.x + element.bbox.width / 2;
+            const centerY = element.bbox.y + element.bbox.height / 2;
+            this.logger?.(`[PageSoMHandler] Using coordinates (${centerX}, ${centerY})`, 'log');
+            const { action, value } = command;
+            let playwrightCommand = '';
+            switch (action) {
+                case som_types_1.InteractionAction.CLICK:
+                case som_types_1.InteractionAction.DOUBLE_CLICK:
+                case som_types_1.InteractionAction.RIGHT_CLICK:
+                    const clickCount = action === som_types_1.InteractionAction.DOUBLE_CLICK ? 2 : 1;
+                    const button = action === som_types_1.InteractionAction.RIGHT_CLICK ? 'right' : 'left';
+                    await this.page.mouse.click(centerX, centerY, { button, clickCount });
+                    playwrightCommand = `await page.mouse.click(${centerX}, ${centerY}${button !== 'left' ? `, { button: '${button}' }` : ''})`;
+                    break;
+                case som_types_1.InteractionAction.FILL:
+                case som_types_1.InteractionAction.TYPE:
+                    // Click first, then type
+                    await this.page.mouse.click(centerX, centerY);
+                    if (action === som_types_1.InteractionAction.FILL) {
+                        await this.page.keyboard.type(value || '');
+                    }
+                    else {
+                        await this.page.keyboard.type(value || '', { delay: command.delay || 50 });
+                    }
+                    playwrightCommand = `await page.mouse.click(${centerX}, ${centerY}); await page.keyboard.type('${value}')`;
+                    break;
+                case som_types_1.InteractionAction.HOVER:
+                    await this.page.mouse.move(centerX, centerY);
+                    playwrightCommand = `await page.mouse.move(${centerX}, ${centerY})`;
+                    break;
+                default:
+                    throw new Error(`Coordinate fallback not supported for action: ${action}`);
+            }
+            return {
+                failedAttempts: [],
+                successAttempt: {
+                    command: playwrightCommand,
+                    status: som_types_1.CommandRunStatus.SUCCESS
+                },
+                status: som_types_1.CommandRunStatus.SUCCESS
+            };
+        }
+        catch (error) {
+            return {
+                failedAttempts: [{
+                        command: `coordinate-based ${command.action}`,
+                        status: som_types_1.CommandRunStatus.FAILURE,
+                        error: error.message
+                    }],
+                error: `Coordinate fallback failed: ${error.message}`,
+                status: som_types_1.CommandRunStatus.FAILURE
+            };
+        }
+    }
+    /**
+     * Setup mutation observer to track DOM changes
+     */
+    async setupMutationObserver() {
+        await this.page.evaluate(() => {
+            const mutations = [];
+            const observer = new MutationObserver((mutationsList) => {
+                for (const mutation of mutationsList) {
+                    if (mutation.type === 'childList') {
+                        mutation.addedNodes.forEach((node) => {
+                            if (node.nodeType === 1) { // Element node
+                                const el = node;
+                                const styles = window.getComputedStyle(el);
+                                // Only track visible additions
+                                if (styles.display !== 'none' && styles.visibility !== 'hidden') {
+                                    mutations.push({
+                                        type: 'added',
+                                        elementDescription: `${el.tagName.toLowerCase()}${el.className ? '.' + el.className.split(' ')[0] : ''}`,
+                                        timestamp: Date.now()
+                                    });
+                                }
+                            }
+                        });
+                    }
+                }
+            });
+            observer.observe(document.body, {
+                childList: true,
+                subtree: true,
+                attributes: true
+            });
+            window.__tcSomMutationObserver = observer;
+            window.__tcSomMutations = mutations;
+        });
+    }
+    /**
+     * Filter relevant mutations (exclude mass updates)
+     */
+    async filterRelevantMutations() {
+        // Get mutations from page
+        const mutations = await this.page.evaluate(() => {
+            return window.__tcSomMutations || [];
+        });
+        // Filter: If too many mutations (>10), likely page rebuild - don't report
+        if (mutations.length > 10) {
+            this.logger?.(`[PageSoMHandler] ${mutations.length} mutations detected - too many, filtering out`, 'warn');
+            return [];
+        }
+        // Return relevant mutations (tooltips, notices, message boxes)
+        return mutations;
+    }
+    /**
+     * Escape selector text
+     */
+    escapeSelector(text) {
+        return text.replace(/'/g, "\\'").trim().substring(0, 50);
+    }
+    /**
+     * Try refined selector with parent scoping
+     */
+    async tryRefinedSelector(typedSelector, command, element, originalError) {
+        // Try parent scoping for generic locators (but avoid if already scoped)
+        if (typedSelector.type === 'locator' && element.parent?.className) {
+            const parentClass = element.parent.className.split(' ')[0];
+            // Guard: Don't add parent if selector already includes it (prevent infinite recursion)
+            if (parentClass && !typedSelector.value.includes(parentClass)) {
+                const refinedSelector = {
+                    type: 'locator',
+                    value: `.${parentClass} ${typedSelector.value}`
+                };
+                const selectorDesc = this.formatSelector(refinedSelector);
+                this.logger?.(`[PageSoMHandler] Trying refined selector with parent: ${selectorDesc}`, 'log');
+                // Important: Don't call tryExecuteAction (would recurse), execute directly
+                try {
+                    const locator = this.buildLocatorFromTypedSelector(refinedSelector);
+                    const refinedDesc = this.formatSelector(refinedSelector);
+                    const playwrightCommand = await this.executeActionOnLocator(locator, command, refinedDesc);
+                    return {
+                        failedAttempts: [{
+                                command: this.formatSelector(typedSelector),
+                                status: som_types_1.CommandRunStatus.FAILURE,
+                                error: originalError.message
+                            }],
+                        successAttempt: {
+                            command: playwrightCommand,
+                            status: som_types_1.CommandRunStatus.SUCCESS
+                        },
+                        status: som_types_1.CommandRunStatus.SUCCESS
+                    };
+                }
+                catch (refinedError) {
+                    // Parent scoping didn't help, fall through to failure
+                    this.logger?.(`[PageSoMHandler] Refined selector also failed: ${refinedError.message}`, 'warn');
+                }
+            }
+        }
+        // Could implement nth() based on bbox position here
+        const selectorDesc = this.formatSelector(typedSelector);
+        return {
+            failedAttempts: [{
+                    command: selectorDesc,
+                    status: som_types_1.CommandRunStatus.FAILURE,
+                    error: originalError.message
+                }],
+            status: som_types_1.CommandRunStatus.FAILURE
+        };
+    }
+}
+exports.PageSoMHandler = PageSoMHandler;
+//# sourceMappingURL=page-som-handler.js.map