testchimp-runner-core 0.0.21 → 0.0.23

package/src/utils/page-info-utils.ts

@@ -1,93 +1,250 @@
-// Page type will be dynamically imported
+/**
+ * Enhanced Page Info Utilities
+ * Extracts ARIA tree, interactive elements with bounding boxes, and suggested selectors
+ */
+
+export interface InteractiveElement {
+  tag: string;
+  id: string | null;
+  text: string;
+  type: string;
+  role: string;
+  ariaLabel: string;
+  placeholder: string;
+  bbox: {x: number; y: number; width: number; height: number};
+  selectors: string[];
+}
 
 export interface PageInfo {
   url: string;
   title: string;
-  elements: string;
-  formFields: string;
-  interactiveElements: string;
-  pageStructure: string;
+  ariaSnapshot: any;  // Raw ARIA tree for hierarchical structure
+  interactiveElements: InteractiveElement[];  // All interactive elements with position and selectors
+  formattedElements: string;  // Human-readable summary for logging
 }
 
 export async function getEnhancedPageInfo(page: any): Promise<PageInfo>;
 export async function getEnhancedPageInfo(domSnapshot: { url: string; title: string; accessibilityTree: any }): Promise<PageInfo>;
 export async function getEnhancedPageInfo(input: any | { url: string; title: string; accessibilityTree: any }): Promise<PageInfo> {
   let domSnapshot: { url: string; title: string; accessibilityTree: any } = { url: 'Unknown', title: 'Unknown', accessibilityTree: null };
+  let interactiveElements: InteractiveElement[] = [];
+  let hasPageObject = false;
   
   try {
     if ('accessibility' in input) {
       // Input is a Page object
+      hasPageObject = true;
       const snapshot = await input.accessibility.snapshot();
       const url = input.url();
       const title = await input.title();
       domSnapshot = { url, title, accessibilityTree: snapshot };
+      
+      // Extract ALL interactive elements with bounding boxes and selectors
+      try {
+        interactiveElements = await input.evaluate(() => {
+          const doc = (globalThis as any).document;
+          const elements: any[] = [];
+          
+          // Get all interactive elements
+          const interactiveSelectors = [
+            'button',
+            'input',
+            'textarea',
+            'select',
+            'a[href]',
+            '[role="button"]',
+            '[role="link"]',
+            '[role="textbox"]',
+            '[role="checkbox"]',
+            '[role="radio"]',
+            '[onclick]'
+          ];
+          
+          const allInteractive = new Set<any>();
+          interactiveSelectors.forEach((selector: string) => {
+            doc.querySelectorAll(selector).forEach((el: any) => allInteractive.add(el));
+          });
+          
+          allInteractive.forEach((el: any) => {
+            const rect = el.getBoundingClientRect();
+            
+            // Skip invisible elements
+            if (rect.width === 0 || rect.height === 0) return;
+            
+            const id = el.id || null;
+            const tag = el.tagName.toLowerCase();
+            const text = el.textContent?.trim().substring(0, 40) || '';
+            const type = el.type || '';
+            const role = el.getAttribute('role') || el.tagName.toLowerCase();
+            const ariaLabel = el.getAttribute('aria-label') || '';
+            const placeholder = el.placeholder || '';
+            
+            // Build possible selectors - PRIORITIZE SEMANTIC SELECTORS (Playwright best practices)
+            const selectors: string[] = [];
+            
+            // 1. BEST: getByLabel for form inputs with associated labels
+            if ((tag === 'input' || tag === 'textarea' || tag === 'select') && el.labels && el.labels.length > 0) {
+              const labelText = el.labels[0].textContent?.trim();
+              if (labelText && labelText.length > 0 && labelText.length < 50) {
+                selectors.push(`getByLabel('${labelText}')`);
+              }
+            }
+            
+            // 2. BEST: getByRole with name (semantic and accessible)
+            if (text && text.length > 0 && text.length < 30) {
+              selectors.push(`getByRole('${role}', {name: '${text}'})`);
+            }
+            
+            // 3. BEST: getByPlaceholder for inputs with placeholders
+            if (placeholder && placeholder.length > 0 && placeholder.length < 50) {
+              selectors.push(`getByPlaceholder('${placeholder}')`);
+            }
+            
+            // 4. GOOD: getByText for elements with visible text (fallback for non-role elements)
+            if (text && text.length > 0 && text.length < 30 && !selectors.some(s => s.includes('getByRole'))) {
+              selectors.push(`getByText('${text}')`);
+            }
+            
+            // 5. GOOD: data-testid attributes (stable, explicit)
+            const dataTestId = el.getAttribute('data-testid') || el.getAttribute('data-test-id') || 
+                              el.getAttribute('data-id') || el.getAttribute('data-test');
+            if (dataTestId) selectors.push(`getByTestId('${dataTestId}')`);
+            
+            // 6. LAST RESORT: ID selector (only if stable - avoid auto-generated IDs with unicode)
+            // Skip auto-generated Ant Design IDs like «r3»-form-item or rc_select_xxx
+            if (id && !id.includes('«') && !id.match(/^(rc_|:r[0-9]+:|__)/)) {
+              selectors.push(`#${id}`);
+            }
+            
+            elements.push({
+              tag,
+              id,
+              text,
+              type,
+              role,
+              ariaLabel,
+              placeholder,
+              bbox: {
+                x: Math.round(rect.x),
+                y: Math.round(rect.y),
+                width: Math.round(rect.width),
+                height: Math.round(rect.height)
+              },
+              selectors
+            });
+          });
+          
+          return elements;
+        });
+      } catch (evalError: any) {
+        // Error extracting interactive elements - continue with empty array
+        console.error('[getEnhancedPageInfo] Failed to extract interactive elements:', evalError.message);
+        interactiveElements = [];
+      }
     } else {
       // Input is already a domSnapshot
       domSnapshot = input;
     }
 
-    // Extract key information from accessibility tree
-    const elements: string[] = [];
-    const formFields: string[] = [];
-    const interactiveElements: string[] = [];
-    const pageStructure: string[] = [];
-
-    const extractElements = (node: any, depth = 0) => {
-      if (depth > 4) return; // Limit depth to avoid overwhelming output
+    // Log extraction results for debugging
+    if (hasPageObject) {
+      console.log(`[getEnhancedPageInfo] Extracted ${interactiveElements.length} interactive elements from ${domSnapshot.url}`);
+    }
+    
+    // Create smart summary prioritizing important elements over repetitive lists
+    let formattedElements = '';
+    
+    if (interactiveElements.length > 0) {
+      // Prioritize: forms/buttons > unique elements > list items
+      const prioritized: InteractiveElement[] = [];
+      const listItems: InteractiveElement[] = [];
       
-      if (node && typeof node === 'object') {
-        if (node.role) {
-          const elementInfo = `${node.role}${node.name ? `: ${node.name}` : ''}`;
-          elements.push(elementInfo);
-
-          // Categorize elements
-          if (['textbox', 'button', 'link', 'checkbox', 'radio', 'combobox', 'slider'].includes(node.role)) {
-            interactiveElements.push(elementInfo);
-          }
-          
-          if (['textbox', 'checkbox', 'radio', 'combobox', 'slider'].includes(node.role)) {
-            formFields.push(elementInfo);
+      // Group elements
+      for (const el of interactiveElements) {
+        // High priority: forms, buttons, unique interactive elements
+        if (['input', 'textarea', 'select', 'button'].includes(el.tag) || 
+            (el.tag === 'a' && el.text.length > 0 && el.text.length < 50)) {
+          prioritized.push(el);
+        } else {
+          listItems.push(el);
+        }
+      }
+      
+      // Take top 25 prioritized
+      const selectedElements = prioritized.slice(0, 25);
+      
+      // Add sample of list items if space remains (show diversity, not just first N)
+      if (selectedElements.length < 30 && listItems.length > 0) {
+        const step = Math.max(1, Math.floor(listItems.length / 5));  // Sample 5 from list
+        for (let i = 0; i < Math.min(5, listItems.length); i++) {
+          selectedElements.push(listItems[i * step]);
+        }
+      }
+      
+      formattedElements = selectedElements.map((e, i) => {
+        const idInfo = e.id ? `#${e.id}` : 'NO-ID';
+        const typeInfo = e.type ? ` type="${e.type}"` : '';
+        const placeholderInfo = e.placeholder ? ` placeholder="${e.placeholder}"` : '';
+        
+        // Show BEST selector first, with alternatives
+        let selectorsInfo: string;
+        if (e.selectors && e.selectors.length > 0) {
+          selectorsInfo = e.selectors[0];  // Best option
+          if (e.selectors.length > 1) {
+            selectorsInfo += ` (or: ${e.selectors.slice(1, 3).join(', ')})`;  // Show up to 2 alternatives
           }
-
-          if (['main', 'navigation', 'banner', 'contentinfo', 'complementary', 'search'].includes(node.role)) {
-            pageStructure.push(elementInfo);
+        } else {
+          selectorsInfo = 'No reliable selector available';
+        }
+        
+        return `${i + 1}. [${e.bbox.x},${e.bbox.y} ${e.bbox.width}x${e.bbox.height}] <${e.tag}> ${idInfo}${typeInfo}${placeholderInfo} "${e.text}"
+   → ${selectorsInfo}`;
+      }).join('\n');
+      
+      // Show stats for omitted elements
+      const omitted = interactiveElements.length - selectedElements.length;
+      if (omitted > 0) {
+        formattedElements += `\n\n... (+${omitted} more elements)`;
+        
+        // Show pattern summary for lists/tables
+        const linkCount = interactiveElements.filter(e => e.tag === 'a').length;
+        const listItemPatterns: Record<string, number> = {};
+        
+        for (const el of interactiveElements) {
+          if (el.tag === 'a' || el.tag === 'li' || el.tag === 'div') {
+            const pattern = `${el.tag}${el.role ? `[${el.role}]` : ''}`;
+            listItemPatterns[pattern] = (listItemPatterns[pattern] || 0) + 1;
           }
         }
-
-        if (node.children) {
-          node.children.forEach((child: any) => extractElements(child, depth + 1));
+        
+        if (Object.keys(listItemPatterns).length > 0) {
+          formattedElements += `\nPattern summary: `;
+          formattedElements += Object.entries(listItemPatterns)
+            .filter(([_, count]) => count > 5)
+            .map(([pattern, count]) => `${count}x ${pattern}`)
+            .join(', ');
         }
       }
-    };
-
-    extractElements(domSnapshot.accessibilityTree);
-
-    // Create a more focused summary
-    const getElementSummary = (elementList: string[], maxItems: number, category: string) => {
-      if (elementList.length === 0) return `No ${category} found`;
-      const limited = elementList.slice(0, maxItems);
-      const remaining = elementList.length - maxItems;
-      const summary = limited.join(', ');
-      return remaining > 0 ? `${summary} (+${remaining} more)` : summary;
-    };
+    } else {
+      formattedElements = 'No interactive elements found (page may not be loaded or extraction failed)';
+    }
 
     return {
       url: domSnapshot.url,
       title: domSnapshot.title,
-      elements: getElementSummary(elements, 15, 'elements'),
-      formFields: getElementSummary(formFields, 8, 'form fields'),
-      interactiveElements: getElementSummary(interactiveElements, 12, 'interactive elements'),
-      pageStructure: getElementSummary(pageStructure, 6, 'page sections')
+      ariaSnapshot: domSnapshot.accessibilityTree,
+      interactiveElements,
+      formattedElements
     };
-  } catch (error) {
-    console.error('Error extracting page info:', error);
+  } catch (error: any) {
+    // Error extracting page info - return defaults
+    console.error('[getEnhancedPageInfo] Top-level error:', error.message);
     return {
       url: domSnapshot?.url || 'Unknown',
       title: 'Unknown',
-      elements: 'Unable to extract',
-      formFields: 'Unable to extract',
-      interactiveElements: 'Unable to extract',
-      pageStructure: 'Unable to extract'
+      ariaSnapshot: null,
+      interactiveElements: [],
+      formattedElements: `Unable to extract: ${error.message}`
     };
   }
 }