testchimp-runner-core 0.0.40 → 0.0.41

package/dist/orchestrator/page-som-handler.js

@@ -47,31 +47,116 @@ class PageSoMHandler {
     /**
      * Update SoM markers - extract interactive elements and draw overlay
      * Clears coordinate markers from previous iteration (agent can view via previous_screenshot tool)
+     * @param includeOffscreen - If true, marks ALL elements including those below viewport (for full-page screenshots)
      */
-    async updateSom() {
-        this.logger?.('[PageSoMHandler] Updating SoM markers...', 'log');
+    async updateSom(includeOffscreen = false) {
+        this.logger?.(`[PageSoMHandler] Updating SoM markers${includeOffscreen ? ' (including offscreen elements)' : ''}...`, 'log');
         // Clear any coordinate markers from previous iteration
         await this.removeCoordinateMarker();
         // Extract interactive elements and assign SoM IDs
-        const elements = await this.page.evaluate(() => {
+        const elements = await this.page.evaluate((includeOffscreen) => {
             const doc = document;
             const elements = [];
             let idCounter = 1;
-            // Query all interactive elements
+            // Helper: Find all shadow roots in the document (recursive, but only called once)
+            function getAllShadowRoots(root = document) {
+                const shadowRoots = [];
+                // Use TreeWalker for efficient DOM traversal (faster than querySelectorAll('*'))
+                const walker = document.createTreeWalker(root, NodeFilter.SHOW_ELEMENT, null);
+                let node = walker.currentNode;
+                while (node) {
+                    if (node.shadowRoot) {
+                        try {
+                            const shadowRoot = node.shadowRoot;
+                            shadowRoots.push(shadowRoot);
+                            // Recursively find shadow roots inside this shadow root
+                            const nestedShadowRoots = getAllShadowRoots(shadowRoot);
+                            shadowRoots.push(...nestedShadowRoots);
+                        }
+                        catch (e) {
+                            // Closed shadow root - skip it
+                        }
+                    }
+                    node = walker.nextNode();
+                }
+                return shadowRoots;
+            }
+            // Helper: Query selector in document + all shadow roots
+            // Much more efficient than recursive approach - finds shadow roots once, then queries each
+            function querySelectorAllDeep(selector) {
+                const results = [];
+                // Query main document
+                doc.querySelectorAll(selector).forEach((el) => results.push(el));
+                // Query each shadow root
+                shadowRoots.forEach(shadowRoot => {
+                    try {
+                        shadowRoot.querySelectorAll(selector).forEach((el) => results.push(el));
+                    }
+                    catch (e) {
+                        // Selector might not be valid in shadow context - skip
+                    }
+                });
+                return results;
+            }
+            // Find all shadow roots upfront (only once, not per selector!)
+            const shadowRoots = getAllShadowRoots(doc);
+            // Query all interactive elements (including shadow DOM)
             const interactiveSelectors = [
                 'button', 'input', 'textarea', 'select', 'a[href]',
                 '[role="button"]', '[role="link"]', '[role="textbox"]',
                 '[role="checkbox"]', '[role="radio"]', '[role="combobox"]',
                 '[role="menu"]', '[role="menuitem"]', '[role="option"]',
-                '[aria-haspopup]', '[onclick]', '[type="submit"]',
+                '[onclick]', '[type="submit"]',
                 '[role="tab"]', '[role="switch"]', '[role="spinbutton"]'
             ];
             const allInteractive = new Set();
             interactiveSelectors.forEach(selector => {
-                doc.querySelectorAll(selector).forEach((el) => allInteractive.add(el));
+                querySelectorAllDeep(selector).forEach((el) => allInteractive.add(el));
+            });
+            // Special handling for containers with ARIA attributes that might not be directly clickable
+            // Pattern 1: [aria-haspopup] - dropdowns, menus, dialogs, date pickers
+            // Pattern 2: [aria-expanded] - accordions, collapses, expandable sections
+            // Pattern 3: [role="combobox"] - custom selects without aria-haspopup
+            const ariaContainerSelectors = ['[aria-haspopup]', '[aria-expanded]', '[role="combobox"]'];
+            ariaContainerSelectors.forEach(selector => {
+                querySelectorAllDeep(selector).forEach((container) => {
+                    // Skip if already marked by other selectors (e.g., button with aria-expanded)
+                    if (allInteractive.has(container))
+                        return;
+                    const styles = window.getComputedStyle(container);
+                    const isContainerClickable = styles.cursor === 'pointer' ||
+                        container.onclick ||
+                        container.getAttribute('onclick') ||
+                        container.tagName === 'BUTTON' ||
+                        container.tagName === 'A';
+                    if (!isContainerClickable) {
+                        // Check for clickable children (buttons, icons, inputs)
+                        const clickableChild = Array.from(container.children).find((child) => {
+                            const childStyles = window.getComputedStyle(child);
+                            return childStyles.cursor === 'pointer' ||
+                                child.onclick ||
+                                child.getAttribute('onclick') ||
+                                child.tagName === 'BUTTON' ||
+                                child.tagName === 'A' ||
+                                child.tagName === 'INPUT';
+                        });
+                        if (clickableChild) {
+                            // Mark the child, not the container
+                            allInteractive.add(clickableChild);
+                        }
+                        else {
+                            // No clickable child found, mark the container as fallback
+                            allInteractive.add(container);
+                        }
+                    }
+                    else {
+                        // Container is clickable, mark it
+                        allInteractive.add(container);
+                    }
+                });
             });
             // Also detect styled-as-interactive elements
-            doc.querySelectorAll('div, span, p, li, td').forEach((el) => {
+            querySelectorAllDeep('div, span, p, li, td').forEach((el) => {
                 const styles = window.getComputedStyle(el);
                 const hasClickHandler = el.onclick || el.getAttribute('onclick') ||
                     el.hasAttribute('data-action') || el.hasAttribute('data-click');
@@ -79,6 +164,22 @@ class PageSoMHandler {
                     allInteractive.add(el);
                 }
             });
+            // Special handling for labels wrapping hidden inputs (custom checkbox/radio patterns)
+            // If a label wraps a hidden input and has visible content, mark the label, not the input
+            querySelectorAllDeep('label').forEach((label) => {
+                const input = label.querySelector('input[type="checkbox"], input[type="radio"]');
+                if (input) {
+                    const inputStyles = window.getComputedStyle(input);
+                    const isInputHidden = inputStyles.display === 'none' ||
+                        inputStyles.visibility === 'hidden' ||
+                        parseFloat(inputStyles.opacity) === 0 ||
+                        (input.getBoundingClientRect().width === 0);
+                    if (isInputHidden) {
+                        // Hidden input with styled label - mark the label instead
+                        allInteractive.add(label);
+                    }
+                }
+            });
             // Remove elements that are descendants of "true" interactive elements
             // (e.g., <span> inside <button> should not get separate marker)
             // But keep elements inside generic containers (div, section, etc.)
@@ -109,8 +210,18 @@ class PageSoMHandler {
                     return;
                 // Skip hidden elements (display, visibility, opacity checks)
                 const styles = window.getComputedStyle(el);
-                if (styles.display === 'none' || styles.visibility === 'hidden' ||
-                    parseFloat(styles.opacity) === 0)
+                const isHidden = styles.display === 'none' ||
+                    (styles.visibility === 'hidden' && parseFloat(styles.opacity) === 0);
+                // Special case: Check for visible pseudo-elements (::before, ::after)
+                // Some sites hide the main element but show content via pseudo-elements
+                let hasVisiblePseudo = false;
+                if (styles.visibility === 'hidden' || parseFloat(styles.opacity) === 0) {
+                    const before = window.getComputedStyle(el, '::before');
+                    const after = window.getComputedStyle(el, '::after');
+                    hasVisiblePseudo = (before.content !== 'none' && before.visibility === 'visible' && before.display !== 'none') ||
+                        (after.content !== 'none' && after.visibility === 'visible' && after.display !== 'none');
+                }
+                if (isHidden && !hasVisiblePseudo)
                     return;
                 // Skip disabled elements (they can't be interacted with)
                 const isDisabled = el.disabled ||
@@ -133,17 +244,30 @@ class PageSoMHandler {
                     { x: rect.left + inset, y: rect.bottom - inset }, // bottom-left
                     { x: rect.right - inset, y: rect.bottom - inset } // bottom-right
                 ];
-                // Element is visible if at least 1 test point hits it or a descendant
-                // (relaxed from 2 to handle small clickable icons better)
-                let visiblePoints = 0;
-                for (const point of testPoints) {
-                    const topEl = document.elementFromPoint(point.x, point.y);
-                    if (topEl && (topEl === el || el.contains(topEl) || topEl.contains(el))) {
-                        visiblePoints++;
+                // Occlusion detection - only for elements in viewport
+                // For offscreen elements (below fold), skip this check since elementFromPoint doesn't work
+                const isInViewport = rect.top < window.innerHeight && rect.bottom > 0 &&
+                    rect.left < window.innerWidth && rect.right > 0;
+                if (!includeOffscreen && isInViewport) {
+                    // Element is in viewport - check if it's occluded
+                    let visiblePoints = 0;
+                    for (const point of testPoints) {
+                        const topEl = document.elementFromPoint(point.x, point.y);
+                        if (topEl && (topEl === el || el.contains(topEl) || topEl.contains(el))) {
+                            visiblePoints++;
+                        }
                     }
+                    // Skip fully occluded elements (no visible points)
+                    if (visiblePoints < 1) {
+                        return;
+                    }
+                }
+                else if (includeOffscreen && !isInViewport) {
+                    // Element is offscreen - include it without occlusion check
+                    // (elementFromPoint doesn't work for offscreen elements)
                 }
-                // Skip fully occluded elements (no visible points)
-                if (visiblePoints < 1) {
+                else if (!includeOffscreen && !isInViewport) {
+                    // Viewport-only mode and element not in viewport - skip it
                     return;
                 }
                 // Assign tc-som-id attribute
@@ -151,12 +275,56 @@ class PageSoMHandler {
                 el.setAttribute('tc-som-id', somId);
                 // Capture element details
                 const parent = el.parentElement;
+                // Extract text - if main element is hidden, try to get text from pseudo-elements
+                let displayText = el.textContent?.trim().substring(0, 50) || '';
+                if (hasVisiblePseudo && (!displayText || styles.visibility === 'hidden')) {
+                    const before = window.getComputedStyle(el, '::before');
+                    const after = window.getComputedStyle(el, '::after');
+                    if (before.content && before.content !== 'none') {
+                        // Remove surrounding quotes from content
+                        displayText = before.content.replace(/^["']|["']$/g, '');
+                    }
+                    else if (after.content && after.content !== 'none') {
+                        displayText = after.content.replace(/^["']|["']$/g, '');
+                    }
+                }
+                // For images, the accessible name comes from alt attribute
+                // For other elements, use aria-label if present
+                let accessibleName = el.getAttribute('aria-label') || '';
+                if (el.tagName.toLowerCase() === 'img' && !accessibleName) {
+                    accessibleName = el.getAttribute('alt') || '';
+                }
+                // Detect associated <label> element for inputs/textareas/selects
+                // This enables getByLabel() selector generation
+                let labelText = '';
+                const tagLower = el.tagName.toLowerCase();
+                if (['input', 'textarea', 'select'].includes(tagLower)) {
+                    // Method 1: Label with for="id"
+                    if (el.id) {
+                        const label = doc.querySelector(`label[for="${el.id}"]`);
+                        if (label) {
+                            labelText = label.textContent?.trim() || '';
+                        }
+                    }
+                    // Method 2: Label wrapping the input
+                    if (!labelText) {
+                        let parent = el.parentElement;
+                        while (parent && parent !== doc.body) {
+                            if (parent.tagName.toLowerCase() === 'label') {
+                                labelText = parent.textContent?.trim() || '';
+                                break;
+                            }
+                            parent = parent.parentElement;
+                        }
+                    }
+                }
                 elements.push({
                     somId,
                     tag: el.tagName.toLowerCase(),
                     role: el.getAttribute('role') || el.tagName.toLowerCase(),
-                    text: el.textContent?.trim().substring(0, 50) || '',
-                    ariaLabel: el.getAttribute('aria-label') || '',
+                    text: displayText,
+                    ariaLabel: accessibleName,
+                    labelText: labelText, // Associated <label> text for getByLabel()
                     placeholder: el.placeholder || '',
                     name: el.getAttribute('name') || '',
                     type: el.type || '',
@@ -168,6 +336,7 @@ class PageSoMHandler {
                         width: Math.round(rect.width),
                         height: Math.round(rect.height)
                     },
+                    hasVisiblePseudoElement: hasVisiblePseudo,
                     parent: parent ? {
                         tag: parent.tagName.toLowerCase(),
                         role: parent.getAttribute('role') || '',
@@ -177,7 +346,7 @@ class PageSoMHandler {
                 });
             });
             return elements;
-        });
+        }, includeOffscreen);
         // Store in somMap
         this.somMap.clear();
         elements.forEach(el => this.somMap.set(el.somId, el));
@@ -457,11 +626,20 @@ class PageSoMHandler {
             await this.setupMutationObserver();
         }
         // Try semantic selectors first (unless useSomIdBasedCommands=true)
+        // For pseudo-element buttons, generateSemanticSelectors will use CSS selectors instead of getByRole/getByText
         if (!useSomIdBasedCommands) {
             const selectors = this.generateSemanticSelectors(element);
             for (let i = 0; i < Math.min(selectors.length, maxAttempts); i++) {
                 const selector = selectors[i];
-                const result = await this.tryExecuteAction(selector, command, element);
+                // Force click/hover for pseudo-element buttons (they have visibility:hidden)
+                const modifiedCommand = element?.hasVisiblePseudoElement &&
+                    (command.action === som_types_1.InteractionAction.CLICK ||
+                        command.action === som_types_1.InteractionAction.DOUBLE_CLICK ||
+                        command.action === som_types_1.InteractionAction.RIGHT_CLICK ||
+                        command.action === som_types_1.InteractionAction.HOVER)
+                    ? { ...command, force: true }
+                    : command;
+                const result = await this.tryExecuteAction(selector, modifiedCommand, element);
                 if (result.status === som_types_1.CommandRunStatus.SUCCESS) {
                     // Wait for mutations if hover/focus
                     if (command.action === som_types_1.InteractionAction.HOVER || command.action === som_types_1.InteractionAction.FOCUS) {
@@ -482,7 +660,15 @@ class PageSoMHandler {
                 type: 'locator',
                 value: `[tc-som-id="${command.elementRef}"]`
             };
-            const result = await this.tryExecuteAction(somIdSelector, command, element);
+            // Force click/interactions for elements with pseudo-elements (they may have visibility:hidden)
+            const modifiedCommand = element?.hasVisiblePseudoElement &&
+                (command.action === som_types_1.InteractionAction.CLICK ||
+                    command.action === som_types_1.InteractionAction.DOUBLE_CLICK ||
+                    command.action === som_types_1.InteractionAction.RIGHT_CLICK ||
+                    command.action === som_types_1.InteractionAction.HOVER)
+                ? { ...command, force: true }
+                : command;
+            const result = await this.tryExecuteAction(somIdSelector, modifiedCommand, element);
             if (result.status === som_types_1.CommandRunStatus.SUCCESS) {
                 await this.disconnectMutationObserver();
                 return result;
@@ -716,12 +902,18 @@ class PageSoMHandler {
         // Priority 1: getByTestId (would need to capture in updateSom)
         // Skipped for now - can add data-testid capture later
         // Priority 2: Stable ID
-        if (element.id && !element.id.match(/^(rc_|:r[0-9]+:|__)/) && !element.id.includes('«')) {
+        // Filter unstable IDs:
+        // - Contains colons (React/MUI: :r123:, :r1r: - need CSS escaping which is fragile)
+        // - Auto-generated prefixes (rc_, __)
+        // - Dynamic markers (contains «)
+        if (element.id && !element.id.includes(':') && !element.id.match(/^(rc_|__)/) && !element.id.includes('«')) {
             selectors.push({ type: 'id', value: element.id });
         }
-        // Priority 3: getByLabel (highly specific for form inputs)
-        if (element.ariaLabel) {
-            selectors.push({ type: 'label', value: element.ariaLabel });
+        // Priority 3: getByLabel (ONLY when there's an actual <label> element)
+        // getByLabel doesn't work with aria-label/alt - it needs <label for="id">
+        // Now properly detected: we check for associated <label> during updateSom()
+        if (element.labelText) {
+            selectors.push({ type: 'label', value: element.labelText });
         }
         // Priority 4: input[name] (backend contract - very stable)
         if (element.name && ['input', 'textarea', 'select'].includes(element.tag)) {
@@ -731,15 +923,36 @@ class PageSoMHandler {
         if (element.placeholder) {
             selectors.push({ type: 'placeholder', value: element.placeholder });
         }
-        // Priority 6: getByRole (broader, can match multiple elements)
-        if (element.role && element.text) {
-            selectors.push({ type: 'role', value: element.role, roleOptions: { name: element.text } });
+        // Priority 6: CSS selector for pseudo-element buttons (BEFORE role/text)
+        // For buttons with visibility:hidden + ::before/::after, getByRole/getByText won't work
+        if (element.hasVisiblePseudoElement && element.tag === 'button') {
+            // Try button[type="submit"] first (most common for forms)
+            if (element.type === 'submit') {
+                selectors.push({ type: 'locator', value: 'button[type="submit"]' });
+            }
+            // Try class-based selector if available
+            if (element.className) {
+                const firstClass = element.className.split(' ')[0];
+                if (firstClass && !firstClass.match(/^(css-|MuiButton-|ant-|btn-)/)) {
+                    selectors.push({ type: 'locator', value: `button.${firstClass}` });
+                }
+            }
+        }
+        // Priority 7: getByRole (broader, can match multiple elements)
+        // Skip for pseudo-element buttons (they have visibility:hidden → not in a11y tree)
+        // For images/buttons with accessible names, prefer ariaLabel over text
+        if (!element.hasVisiblePseudoElement && element.role) {
+            const accessibleName = element.ariaLabel || element.text;
+            if (accessibleName) {
+                selectors.push({ type: 'role', value: element.role, roleOptions: { name: accessibleName } });
+            }
         }
-        // Priority 7: getByText (last semantic option)
-        if (element.text) {
+        // Priority 8: getByText (last semantic option)
+        // Skip for pseudo-element buttons (getByText will match the heading instead!)
+        if (!element.hasVisiblePseudoElement && element.text) {
             selectors.push({ type: 'text', value: element.text });
         }
-        // Priority 8: Parent-scoped locator (generic fallback)
+        // Priority 9: Parent-scoped locator (generic fallback)
         if (element.parent?.className) {
             const parentClass = element.parent.className.split(' ')[0];
             if (parentClass) {
@@ -1194,6 +1407,10 @@ class PageSoMHandler {
                     await this.page.mouse.up();
                     playwrightCommand = `await page.mouse.move(${pixelX}, ${pixelY}); await page.mouse.down(); await page.mouse.move(${toPixelX}, ${toPixelY}); await page.mouse.up()`;
                     break;
+                case som_types_1.InteractionAction.PRESS:
+                case som_types_1.InteractionAction.PRESS_SEQUENTIALLY:
+                    // Keyboard press is page-level - coordinates don't make sense
+                    throw new Error(`PRESS action cannot use coordinates. To scroll: use SCROLL action with scrollDirection/scrollAmount. To press keys: use PRESS without coord.`);
                 default:
                     throw new Error(`Coordinate-based execution not supported for action: ${action}`);
             }