playwriter 0.0.38 → 0.0.40

package/src/aria-snapshot.ts

@@ -1,4 +1,7 @@
 import type { Page, Locator, ElementHandle } from 'playwright-core'
+import fs from 'node:fs'
+import path from 'node:path'
+import os from 'node:os'
 
 export interface AriaRef {
   role: string
@@ -6,6 +9,14 @@ export interface AriaRef {
   ref: string
 }
 
+export interface ScreenshotResult {
+  path: string
+  base64: string
+  mimeType: 'image/jpeg'
+  snapshot: string
+  labelCount: number
+}
+
 export interface AriaSnapshotResult {
   snapshot: string
   refToElement: Map<string, { role: string; name: string }>
@@ -17,7 +28,7 @@ export interface AriaSnapshotResult {
 
 const LABELS_CONTAINER_ID = '__playwriter_labels__'
 
-// Roles that represent truly interactive elements (can be clicked, typed into, etc.)
+// Roles that represent interactive elements (clickable, typeable) and media elements
 const INTERACTIVE_ROLES = new Set([
   'button',
   'link',
@@ -35,6 +46,10 @@ const INTERACTIVE_ROLES = new Set([
   'option',
   'tab',
   'treeitem',
+  // Media elements - useful for visual tasks
+  'img',
+  'video',
+  'audio',
 ])
 
 // Color categories for different role types - warm color scheme
@@ -63,6 +78,10 @@ const ROLE_COLORS: Record<string, [string, string, string]> = {
   tab: ['#FFE082', '#FFD54F', '#FFC107'],
   option: ['#FFE082', '#FFD54F', '#FFC107'],
   treeitem: ['#FFE082', '#FFD54F', '#FFC107'],
+  // Media elements - light blue
+  img: ['#B3E5FC', '#81D4FA', '#4FC3F7'],
+  video: ['#B3E5FC', '#81D4FA', '#4FC3F7'],
+  audio: ['#B3E5FC', '#81D4FA', '#4FC3F7'],
 }
 
 // Default yellow for unknown roles
@@ -197,7 +216,7 @@ export async function getAriaSnapshot({ page }: { page: Page }): Promise<AriaSna
  * Labels are yellow badges positioned above each element showing the aria ref (e.g., "e1", "e2").
  * Use with screenshots so agents can see which elements are interactive.
  *
- * Labels auto-hide after 5 seconds to prevent stale labels remaining on the page.
+ * Labels auto-hide after 30 seconds to prevent stale labels remaining on the page.
  * Call this function again if the page HTML changes to get fresh labels.
  *
  * By default, only shows labels for truly interactive roles (button, link, textbox, etc.)
@@ -209,7 +228,7 @@ export async function getAriaSnapshot({ page }: { page: Page }): Promise<AriaSna
  * await page.screenshot({ path: '/tmp/screenshot.png' })
  * // Agent sees [e5] label on "Submit" button
  * await page.locator('aria-ref=e5').click()
- * // Labels auto-hide after 5 seconds, or call hideAriaRefLabels() manually
+ * // Labels auto-hide after 30 seconds, or call hideAriaRefLabels() manually
  * ```
  */
 export async function showAriaRefLabels({ page, interactiveOnly = true }: {
@@ -238,10 +257,9 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
 
   // Single evaluate call: create container, styles, and all labels
   // ElementHandles get unwrapped to DOM elements in browser context
-  // Using 'any' types here since this code runs in browser context
   const labelCount = await page.evaluate(
     // Using 'any' for browser types since this runs in browser context
-    ({ refs, containerId, containerStyles, labelStyles, roleColors, defaultColors }: {
+    function ({ refs, containerId, containerStyles, labelStyles, roleColors, defaultColors }: {
       refs: Array<{
         ref: string
         role: string
@@ -252,7 +270,9 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
       labelStyles: string
       roleColors: Record<string, [string, string, string]>
       defaultColors: [string, string, string]
-    }) => {
+    }) {
+      // Polyfill esbuild's __name helper which gets injected by vite-node but doesn't exist in browser
+      ;(globalThis as any).__name ||= (fn: any) => fn
       const doc = (globalThis as any).document
       const win = globalThis as any
 
@@ -285,8 +305,10 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
       const LABEL_CHAR_WIDTH = 7 // approximate width per character
 
       // Parse alpha from rgb/rgba color string (getComputedStyle always returns these formats)
-      const getColorAlpha = (color: string): number => {
-        if (color === 'transparent') return 0
+      function getColorAlpha(color: string): number {
+        if (color === 'transparent') {
+          return 0
+        }
         // Match rgba(r, g, b, a) or rgb(r, g, b)
         const match = color.match(/rgba?\(\s*[\d.]+\s*,\s*[\d.]+\s*,\s*[\d.]+\s*(?:,\s*([\d.]+)\s*)?\)/)
         if (match) {
@@ -296,25 +318,31 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
       }
 
       // Check if an element has an opaque background that would block elements behind it
-      const isOpaqueElement = (el: any): boolean => {
+      function isOpaqueElement(el: any): boolean {
         const style = win.getComputedStyle(el)
 
         // Check element opacity
         const opacity = parseFloat(style.opacity)
-        if (opacity < 0.1) return false
+        if (opacity < 0.1) {
+          return false
+        }
 
         // Check background-color alpha
         const bgAlpha = getColorAlpha(style.backgroundColor)
-        if (bgAlpha > 0.1) return true
+        if (bgAlpha > 0.1) {
+          return true
+        }
 
         // Check if has background-image (usually opaque)
-        if (style.backgroundImage !== 'none') return true
+        if (style.backgroundImage !== 'none') {
+          return true
+        }
 
         return false
       }
 
       // Check if element is visible (not covered by opaque overlay)
-      const isElementVisible = (element: any, rect: any): boolean => {
+      function isElementVisible(element: any, rect: any): boolean {
         const centerX = rect.left + rect.width / 2
         const centerY = rect.top + rect.height / 2
 
@@ -322,32 +350,44 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
         const stack = doc.elementsFromPoint(centerX, centerY) as any[]
 
         // Find our target element in the stack
-        const targetIndex = stack.findIndex((el: any) =>
-          element.contains(el) || el.contains(element)
-        )
+        let targetIndex = -1
+        for (let i = 0; i < stack.length; i++) {
+          if (element.contains(stack[i]) || stack[i].contains(element)) {
+            targetIndex = i
+            break
+          }
+        }
 
         // Element not in stack at all - not visible
-        if (targetIndex === -1) return false
+        if (targetIndex === -1) {
+          return false
+        }
 
         // Check if any opaque element is above our target
         for (let i = 0; i < targetIndex; i++) {
           const el = stack[i]
           // Skip our own overlay container
-          if (el.id === containerId) continue
+          if (el.id === containerId) {
+            continue
+          }
           // Skip pointer-events: none elements (decorative overlays)
-          if (win.getComputedStyle(el).pointerEvents === 'none') continue
+          if (win.getComputedStyle(el).pointerEvents === 'none') {
+            continue
+          }
           // If this element is opaque, our target is blocked
-          if (isOpaqueElement(el)) return false
+          if (isOpaqueElement(el)) {
+            return false
+          }
         }
 
         return true
       }
 
       // Check if two rectangles overlap
-      const rectsOverlap = (
+      function rectsOverlap(
         a: { left: number; top: number; right: number; bottom: number },
         b: { left: number; top: number; right: number; bottom: number }
-      ) => {
+      ) {
         return a.left < b.right && a.right > b.left && a.top < b.bottom && a.bottom > b.top
       }
 
@@ -378,7 +418,13 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
         }
 
         // Skip if this label would overlap with any already-placed label
-        const overlaps = placedLabels.some((placed) => rectsOverlap(labelRect, placed))
+        let overlaps = false
+        for (const placed of placedLabels) {
+          if (rectsOverlap(labelRect, placed)) {
+            overlaps = true
+            break
+          }
+        }
         if (overlaps) {
           continue
         }
@@ -404,12 +450,12 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
 
       doc.documentElement.appendChild(container)
 
-      // Auto-hide labels after 5 seconds to prevent stale labels
+      // Auto-hide labels after 30 seconds to prevent stale labels
       // Store timer ID so it can be cancelled if showAriaRefLabels is called again
-      win[timerKey] = win.setTimeout(() => {
+      win[timerKey] = win.setTimeout(function() {
         doc.getElementById(containerId)?.remove()
         win[timerKey] = null
-      }, 5000)
+      }, 30000)
 
       return count
     },
@@ -444,3 +490,60 @@ export async function hideAriaRefLabels({ page }: { page: Page }): Promise<void>
     doc.getElementById(id)?.remove()
   }, LABELS_CONTAINER_ID)
 }
+
+/**
+ * Take a screenshot with accessibility labels overlaid on interactive elements.
+ * Shows Vimium-style labels, captures the screenshot, then removes the labels.
+ * The screenshot is automatically included in the MCP response.
+ *
+ * @param collector - Array to collect screenshots (passed by MCP execute tool)
+ *
+ * @example
+ * ```ts
+ * await screenshotWithAccessibilityLabels({ page })
+ * // Screenshot is automatically included in the MCP response
+ * // Use aria-ref from the snapshot to interact with elements
+ * await page.locator('aria-ref=e5').click()
+ * ```
+ */
+export async function screenshotWithAccessibilityLabels({ page, interactiveOnly = true, collector }: {
+  page: Page
+  interactiveOnly?: boolean
+  collector: ScreenshotResult[]
+}): Promise<void> {
+  // Show labels and get snapshot
+  const { snapshot, labelCount } = await showAriaRefLabels({ page, interactiveOnly })
+
+  // Generate unique filename with timestamp
+  const timestamp = Date.now()
+  const random = Math.random().toString(36).slice(2, 6)
+  const filename = `playwriter-screenshot-${timestamp}-${random}.jpg`
+  
+  // Use ./tmp folder (gitignored) instead of system temp
+  const tmpDir = path.join(process.cwd(), 'tmp')
+  if (!fs.existsSync(tmpDir)) {
+    fs.mkdirSync(tmpDir, { recursive: true })
+  }
+  const screenshotPath = path.join(tmpDir, filename)
+
+  // Take screenshot
+  const buffer = await page.screenshot({ type: 'jpeg', quality: 80 })
+  
+  // Save to file
+  fs.writeFileSync(screenshotPath, buffer)
+  
+  // Convert to base64
+  const base64 = buffer.toString('base64')
+
+  // Hide labels
+  await hideAriaRefLabels({ page })
+
+  // Add to collector array
+  collector.push({
+    path: screenshotPath,
+    base64,
+    mimeType: 'image/jpeg',
+    snapshot,
+    labelCount,
+  })
+}

package/src/assets/aria-labels-google-snapshot.txt

@@ -31,7 +31,7 @@
     - generic [ref=e55]:
       - text: "Google offered in:"
       - link [ref=e56] [cursor=pointer]:
-        - /url: https://www.google.com/setprefs?sig=0_LaNB9MnwJduON19vFVmxHtMW8aY%3D&hl=it&source=homepage&sa=X&ved=0ahUKEwj22fSr2--RAxXoGTQIHVbSDAcQ2ZgBCBU
+        - /url: https://www.google.com/setprefs?sig=0_U_LREl4THfXa2sQ-b49g0Jd45jk%3D&hl=it&source=homepage&sa=X&ved=0ahUKEwiYgJn4hfSRAxU00xoGHUTZBMEQ2ZgBCBU
         - text: Italiano
     - contentinfo [ref=e58]:
       - generic [ref=e59]: Italy