playwriter 0.0.39 → 0.0.41

package/src/aria-snapshot.ts

@@ -1,4 +1,7 @@
 import type { Page, Locator, ElementHandle } from 'playwright-core'
+import fs from 'node:fs'
+import path from 'node:path'
+import os from 'node:os'
 
 export interface AriaRef {
   role: string
@@ -6,6 +9,14 @@ export interface AriaRef {
   ref: string
 }
 
+export interface ScreenshotResult {
+  path: string
+  base64: string
+  mimeType: 'image/jpeg'
+  snapshot: string
+  labelCount: number
+}
+
 export interface AriaSnapshotResult {
   snapshot: string
   refToElement: Map<string, { role: string; name: string }>
@@ -17,7 +28,7 @@ export interface AriaSnapshotResult {
 
 const LABELS_CONTAINER_ID = '__playwriter_labels__'
 
-// Roles that represent truly interactive elements (can be clicked, typed into, etc.)
+// Roles that represent interactive elements (clickable, typeable) and media elements
 const INTERACTIVE_ROLES = new Set([
   'button',
   'link',
@@ -35,6 +46,10 @@ const INTERACTIVE_ROLES = new Set([
   'option',
   'tab',
   'treeitem',
+  // Media elements - useful for visual tasks
+  'img',
+  'video',
+  'audio',
 ])
 
 // Color categories for different role types - warm color scheme
@@ -63,6 +78,10 @@ const ROLE_COLORS: Record<string, [string, string, string]> = {
   tab: ['#FFE082', '#FFD54F', '#FFC107'],
   option: ['#FFE082', '#FFD54F', '#FFC107'],
   treeitem: ['#FFE082', '#FFD54F', '#FFC107'],
+  // Media elements - light blue
+  img: ['#B3E5FC', '#81D4FA', '#4FC3F7'],
+  video: ['#B3E5FC', '#81D4FA', '#4FC3F7'],
+  audio: ['#B3E5FC', '#81D4FA', '#4FC3F7'],
 }
 
 // Default yellow for unknown roles
@@ -74,7 +93,7 @@ const css = String.raw
 const LABEL_STYLES = css`
   .__pw_label__ {
     position: absolute;
-    font: bold 11px Helvetica, Arial, sans-serif;
+    font: bold 12px Helvetica, Arial, sans-serif;
     padding: 1px 4px;
     border-radius: 3px;
     color: black;
@@ -281,8 +300,8 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
       // Each rect is { left, top, right, bottom } in viewport coordinates
       const placedLabels: Array<{ left: number; top: number; right: number; bottom: number }> = []
 
-      // Estimate label dimensions (11px font + padding)
-      const LABEL_HEIGHT = 16
+      // Estimate label dimensions (12px font + padding)
+      const LABEL_HEIGHT = 17
       const LABEL_CHAR_WIDTH = 7 // approximate width per character
 
       // Parse alpha from rgb/rgba color string (getComputedStyle always returns these formats)
@@ -372,6 +391,41 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
         return a.left < b.right && a.right > b.left && a.top < b.bottom && a.bottom > b.top
       }
 
+      // Create SVG for connector lines
+      const svg = doc.createElementNS('http://www.w3.org/2000/svg', 'svg')
+      svg.style.cssText = 'position:absolute;left:0;top:0;pointer-events:none;overflow:visible;'
+      svg.setAttribute('width', `${doc.documentElement.scrollWidth}`)
+      svg.setAttribute('height', `${doc.documentElement.scrollHeight}`)
+
+      // Create defs for arrow markers (one per color)
+      const defs = doc.createElementNS('http://www.w3.org/2000/svg', 'defs')
+      svg.appendChild(defs)
+      const markerCache: Record<string, string> = {}
+
+      function getArrowMarkerId(color: string): string {
+        if (markerCache[color]) {
+          return markerCache[color]
+        }
+        const markerId = `arrow-${color.replace('#', '')}`
+        const marker = doc.createElementNS('http://www.w3.org/2000/svg', 'marker')
+        marker.setAttribute('id', markerId)
+        marker.setAttribute('viewBox', '0 0 10 10')
+        marker.setAttribute('refX', '9')
+        marker.setAttribute('refY', '5')
+        marker.setAttribute('markerWidth', '6')
+        marker.setAttribute('markerHeight', '6')
+        marker.setAttribute('orient', 'auto-start-reverse')
+        const path = doc.createElementNS('http://www.w3.org/2000/svg', 'path')
+        path.setAttribute('d', 'M 0 0 L 10 5 L 0 10 z')
+        path.setAttribute('fill', color)
+        marker.appendChild(path)
+        defs.appendChild(marker)
+        markerCache[color] = markerId
+        return markerId
+      }
+
+      container.appendChild(svg)
+
       // Create label for each interactive element
       let count = 0
       for (const { ref, role, element } of refs) {
@@ -425,6 +479,22 @@ export async function showAriaRefLabels({ page, interactiveOnly = true }: {
         label.style.top = `${win.scrollY + labelTop}px`
 
         container.appendChild(label)
+
+        // Draw connector line from label bottom-center to element center with arrow
+        const line = doc.createElementNS('http://www.w3.org/2000/svg', 'line')
+        const labelCenterX = win.scrollX + labelLeft + labelWidth / 2
+        const labelBottomY = win.scrollY + labelTop + LABEL_HEIGHT
+        const elementCenterX = win.scrollX + rect.left + rect.width / 2
+        const elementCenterY = win.scrollY + rect.top + rect.height / 2
+        line.setAttribute('x1', `${labelCenterX}`)
+        line.setAttribute('y1', `${labelBottomY}`)
+        line.setAttribute('x2', `${elementCenterX}`)
+        line.setAttribute('y2', `${elementCenterY}`)
+        line.setAttribute('stroke', border)
+        line.setAttribute('stroke-width', '1.5')
+        line.setAttribute('marker-end', `url(#${getArrowMarkerId(border)})`)
+        svg.appendChild(line)
+
         placedLabels.push(labelRect)
         count++
       }
@@ -471,3 +541,68 @@ export async function hideAriaRefLabels({ page }: { page: Page }): Promise<void>
     doc.getElementById(id)?.remove()
   }, LABELS_CONTAINER_ID)
 }
+
+/**
+ * Take a screenshot with accessibility labels overlaid on interactive elements.
+ * Shows Vimium-style labels, captures the screenshot, then removes the labels.
+ * The screenshot is automatically included in the MCP response.
+ *
+ * @param collector - Array to collect screenshots (passed by MCP execute tool)
+ *
+ * @example
+ * ```ts
+ * await screenshotWithAccessibilityLabels({ page })
+ * // Screenshot is automatically included in the MCP response
+ * // Use aria-ref from the snapshot to interact with elements
+ * await page.locator('aria-ref=e5').click()
+ * ```
+ */
+export async function screenshotWithAccessibilityLabels({ page, interactiveOnly = true, collector }: {
+  page: Page
+  interactiveOnly?: boolean
+  collector: ScreenshotResult[]
+}): Promise<void> {
+  // Show labels and get snapshot
+  const { snapshot, labelCount } = await showAriaRefLabels({ page, interactiveOnly })
+
+  // Generate unique filename with timestamp
+  const timestamp = Date.now()
+  const random = Math.random().toString(36).slice(2, 6)
+  const filename = `playwriter-screenshot-${timestamp}-${random}.jpg`
+
+  // Use ./tmp folder (gitignored) instead of system temp
+  const tmpDir = path.join(process.cwd(), 'tmp')
+  if (!fs.existsSync(tmpDir)) {
+    fs.mkdirSync(tmpDir, { recursive: true })
+  }
+  const screenshotPath = path.join(tmpDir, filename)
+
+  // Get actual viewport size (innerWidth/innerHeight, not outer window size)
+  const viewport = await page.evaluate('(() => ({ width: window.innerWidth, height: window.innerHeight }))()') as { width: number; height: number }
+
+  // Take screenshot clipped to actual viewport (excludes browser chrome)
+  const buffer = await page.screenshot({
+    type: 'jpeg',
+    quality: 80,
+    scale: 'css',
+    clip: { x: 0, y: 0, width: viewport.width, height: viewport.height },
+  })
+
+  // Save to file
+  fs.writeFileSync(screenshotPath, buffer)
+
+  // Convert to base64
+  const base64 = buffer.toString('base64')
+
+  // Hide labels
+  await hideAriaRefLabels({ page })
+
+  // Add to collector array
+  collector.push({
+    path: screenshotPath,
+    base64,
+    mimeType: 'image/jpeg',
+    snapshot,
+    labelCount,
+  })
+}

package/src/assets/aria-labels-google-snapshot.txt

@@ -31,7 +31,7 @@
     - generic [ref=e55]:
       - text: "Google offered in:"
       - link [ref=e56] [cursor=pointer]:
-        - /url: https://www.google.com/setprefs?sig=0_LaNB9MnwJduON19vFVmxHtMW8aY%3D&hl=it&source=homepage&sa=X&ved=0ahUKEwj22fSr2--RAxXoGTQIHVbSDAcQ2ZgBCBU
+        - /url: https://www.google.com/setprefs?sig=0_XgFjFgidjiUAiTgTlmWfO1H5x10%3D&hl=it&source=homepage&sa=X&ved=0ahUKEwjT-5m25PaRAxUON94AHdLuNggQ2ZgBCBU
         - text: Italiano
     - contentinfo [ref=e58]:
       - generic [ref=e59]: Italy