playwriter 0.0.89 → 0.0.103

package/src/aria-snapshot.ts

@@ -79,7 +79,7 @@ export type AriaSnapshotNode = {
 export interface ScreenshotResult {
   path: string
   base64: string
-  mimeType: 'image/jpeg'
+  mimeType: 'image/png'
   snapshot: string
   labelCount: number
 }
@@ -108,13 +108,15 @@ export interface ResizeImageOptions {
   fit?: 'inside' | 'cover' | 'contain' | 'fill'
   /** JPEG quality 1-100. Default 80 */
   quality?: number
+  /** Output format. Default 'jpeg'. Use 'png' for lossless output (e.g. Kitty Graphics). */
+  format?: 'jpeg' | 'png'
   /** Output file path. Defaults to overwriting the input file (when input is a path) */
   output?: string
 }
 
 export interface ResizeImageResult {
   buffer: Buffer
-  mimeType: 'image/jpeg'
+  mimeType: 'image/png' | 'image/jpeg'
   /** Only set if output path was provided */
   path?: string
 }
@@ -128,7 +130,7 @@ export interface ResizeImageResult {
  *
  * Explicit width/height: resizes to those dimensions using the fit strategy.
  */
-export async function resizeImage(options: ResizeImageOptions): Promise<ResizeImageResult> {
+export async function resizeImageForAgent(options: ResizeImageOptions): Promise<ResizeImageResult> {
   const sharp = await sharpPromise
   if (!sharp) {
     throw new Error('sharp is not installed — install it with: pnpm add sharp')
@@ -163,10 +165,9 @@ export async function resizeImage(options: ResizeImageOptions): Promise<ResizeIm
     }
   })()
 
-  const buffer = await sharp(inputBuffer)
-    .resize(resizeOpts)
-    .jpeg({ quality })
-    .toBuffer()
+  const fmt = options.format ?? 'png'
+  const pipeline = sharp(inputBuffer).resize(resizeOpts)
+  const buffer = await (fmt === 'png' ? pipeline.png() : pipeline.jpeg({ quality })).toBuffer()
 
   // Default: overwrite input file. When input is a Buffer, no file is written
   // unless output is explicitly set.
@@ -184,9 +185,10 @@ export async function resizeImage(options: ResizeImageOptions): Promise<ResizeIm
     fs.writeFileSync(outputPath, buffer)
   }
 
+  const mimeType: 'image/png' | 'image/jpeg' = fmt === 'png' ? 'image/png' : 'image/jpeg'
   return {
     buffer,
-    mimeType: 'image/jpeg',
+    mimeType,
     ...(outputPath ? { path: outputPath } : {}),
   }
 }
@@ -338,14 +340,24 @@ function buildBaseLocator({
   role,
   name,
   stable,
+  isPromotedContentEditable,
 }: {
   role: string
   name: string
   stable: { value: string; attr: string } | null
+  isPromotedContentEditable?: boolean
 }): string {
   if (stable) {
     return buildLocatorFromStable(stable)
   }
+  // For promoted contenteditable elements (bare <div contenteditable="true"> without
+  // role="textbox"), use CSS attribute selector. Playwright's role=textbox selector
+  // won't match these because Chrome doesn't assign an implicit textbox role to
+  // contenteditable divs. Elements that already had role="textbox" use the normal
+  // role-based locator since Playwright matches those correctly.
+  if (isPromotedContentEditable) {
+    return `[contenteditable="true"]`
+  }
   const trimmedName = name.trim()
   if (trimmedName.length > 0) {
     const escapedName = escapeLocatorValue(trimmedName)
@@ -486,6 +498,7 @@ export function filterInteractiveSnapshotTree(options: {
   labelContext: boolean
   refFilter?: (entry: { role: string; name: string }) => boolean
   domByBackendId: Map<Protocol.DOM.BackendNodeId, DomNodeInfo>
+  promotedContentEditableIds?: Set<Protocol.DOM.BackendNodeId>
   createRefForNode: (options: {
     backendNodeId?: Protocol.DOM.BackendNodeId
     role: string
@@ -507,6 +520,7 @@ export function filterInteractiveSnapshotTree(options: {
       labelContext: nextLabelContext,
       refFilter: options.refFilter,
       domByBackendId: options.domByBackendId,
+      promotedContentEditableIds: options.promotedContentEditableIds,
       createRefForNode: options.createRefForNode,
     })
   })
@@ -575,7 +589,8 @@ export function filterInteractiveSnapshotTree(options: {
   if (includeInteractive) {
     const domInfo = options.node.backendNodeId ? options.domByBackendId.get(options.node.backendNodeId) : undefined
     const stable = domInfo ? getStableRefFromAttributes(domInfo.attributes) : null
-    baseLocator = buildBaseLocator({ role, name, stable })
+    const isPromoted = options.node.backendNodeId != null && (options.promotedContentEditableIds?.has(options.node.backendNodeId) ?? false)
+    baseLocator = buildBaseLocator({ role, name, stable, isPromotedContentEditable: isPromoted })
     ref = options.createRefForNode({ backendNodeId: options.node.backendNodeId, role, name })
   }
 
@@ -599,6 +614,7 @@ export function filterFullSnapshotTree(options: {
   ancestorNames: string[]
   refFilter?: (entry: { role: string; name: string }) => boolean
   domByBackendId: Map<Protocol.DOM.BackendNodeId, DomNodeInfo>
+  promotedContentEditableIds?: Set<Protocol.DOM.BackendNodeId>
   createRefForNode: (options: {
     backendNodeId?: Protocol.DOM.BackendNodeId
     role: string
@@ -616,6 +632,7 @@ export function filterFullSnapshotTree(options: {
       ancestorNames: nextAncestors,
       refFilter: options.refFilter,
       domByBackendId: options.domByBackendId,
+      promotedContentEditableIds: options.promotedContentEditableIds,
       createRefForNode: options.createRefForNode,
     })
   })
@@ -673,7 +690,8 @@ export function filterFullSnapshotTree(options: {
   if (includeInteractive) {
     const domInfo = options.node.backendNodeId ? options.domByBackendId.get(options.node.backendNodeId) : undefined
     const stable = domInfo ? getStableRefFromAttributes(domInfo.attributes) : null
-    baseLocator = buildBaseLocator({ role, name, stable })
+    const isPromoted = options.node.backendNodeId != null && (options.promotedContentEditableIds?.has(options.node.backendNodeId) ?? false)
+    baseLocator = buildBaseLocator({ role, name, stable, isPromotedContentEditable: isPromoted })
     ref = options.createRefForNode({ backendNodeId: options.node.backendNodeId, role, name })
   }
 
@@ -853,6 +871,17 @@ function isTextRole(role: string): boolean {
   return role === 'statictext' || role === 'inlinetextbox'
 }
 
+// Per HTML spec, contenteditable is editable when the attribute is present
+// with value "true", "" (empty string), or "plaintext-only". Bare attribute
+// (no value) is also treated as empty string by the browser DOM parser.
+function isContentEditable(value: string | undefined | null): boolean {
+  if (value == null) {
+    return false
+  }
+  const v = value.trim().toLowerCase()
+  return v === '' || v === 'true' || v === 'plaintext-only'
+}
+
 function isSubstringOfAny(needle: string, haystack: Set<string>): boolean {
   for (const str of haystack) {
     if (str.includes(needle)) {
@@ -1017,11 +1046,41 @@ export async function getAriaSnapshot({
       axById.set(node.nodeId, node)
     }
 
+    // Index AX nodes by backendDOMNodeId for O(1) lookups during promotion
+    // and root finding (instead of repeated O(n) axNodes.find() calls)
+    const axByBackendId = new Map<Protocol.DOM.BackendNodeId, Protocol.Accessibility.AXNode>()
+    for (const node of axNodes) {
+      if (node.backendDOMNodeId) {
+        axByBackendId.set(node.backendDOMNodeId, node)
+      }
+    }
+
+    // Promote contenteditable elements that Chrome's AX tree doesn't classify as
+    // interactive. Rich text editors (ProseMirror, Tiptap, Slate, Lexical, etc.) use
+    // bare <div contenteditable="true"> without role="textbox", so Chrome reports
+    // them as "generic" and they become invisible in the snapshot. We detect these
+    // via the DOM tree and override the AX role to "textbox" so they appear as
+    // interactive elements the AI can target.
+    const promotedContentEditableIds = new Set<Protocol.DOM.BackendNodeId>()
+    for (const [, domInfo] of domByBackendId) {
+      if (!isContentEditable(domInfo.attributes.get('contenteditable'))) {
+        continue
+      }
+      const axNode = axByBackendId.get(domInfo.backendNodeId)
+      if (!axNode) {
+        continue
+      }
+      const currentRole = getAxRole(axNode)
+      if (INTERACTIVE_ROLES.has(currentRole)) {
+        continue
+      }
+      axNode.role = { type: 'role', value: 'textbox' }
+      promotedContentEditableIds.add(domInfo.backendNodeId)
+    }
+
     const findRootAxNodeId = (): Protocol.Accessibility.AXNodeId | null => {
       if (scopeRootBackendId) {
-        const scoped = axNodes.find((node) => {
-          return node.backendDOMNodeId === scopeRootBackendId
-        })
+        const scoped = axByBackendId.get(scopeRootBackendId)
         if (scoped) {
           return scoped.nodeId
         }
@@ -1075,6 +1134,12 @@ export async function getAriaSnapshot({
       if (stable && count === 0) {
         selector = buildLocatorFromStable(stable)
       }
+      // For promoted contenteditable elements without a stable selector, store
+      // [contenteditable="true"] so getSelectorForRef() doesn't fall back to
+      // role=textbox which Playwright can't match on bare contenteditable divs.
+      if (!selector && options.backendNodeId != null && promotedContentEditableIds.has(options.backendNodeId)) {
+        selector = '[contenteditable="true"]'
+      }
 
       refs.push({ ref, role: options.role, name: options.name, selector, backendNodeId: options.backendNodeId })
       return ref
@@ -1111,6 +1176,7 @@ export async function getAriaSnapshot({
             labelContext: false,
             refFilter,
             domByBackendId,
+            promotedContentEditableIds,
             createRefForNode,
           }).nodes
         }
@@ -1119,6 +1185,7 @@ export async function getAriaSnapshot({
           ancestorNames: [],
           refFilter,
           domByBackendId,
+          promotedContentEditableIds,
           createRefForNode,
         }).nodes
       })
@@ -1538,7 +1605,7 @@ export async function screenshotWithAccessibilityLabels({
   // Generate unique filename with timestamp
   const timestamp = Date.now()
   const random = Math.random().toString(36).slice(2, 6)
-  const filename = `playwriter-screenshot-${timestamp}-${random}.jpg`
+  const filename = `playwriter-screenshot-${timestamp}-${random}.png`
 
   // Use ./tmp folder (gitignored) instead of system temp
   const tmpDir = path.join(process.cwd(), 'tmp')
@@ -1560,11 +1627,11 @@ export async function screenshotWithAccessibilityLabels({
   const clipWidth = sharp ? viewport.width : Math.min(viewport.width, LLM_MAX_DIMENSION)
   const clipHeight = sharp ? viewport.height : Math.min(viewport.height, LLM_MAX_DIMENSION)
 
-  // Take viewport screenshot with scale: 'css' to ignore device pixel ratio
+  // Take viewport screenshot as PNG for Kitty Graphics Protocol compatibility.
+  // PNG is lossless and the only format extracted by kitty-graphics-agent (f=100).
   const screenshotStart = Date.now()
   const rawBuffer = await page.screenshot({
-    type: 'jpeg',
-    quality: 80,
+    type: 'png',
     scale: 'css',
     clip: { x: 0, y: 0, width: clipWidth, height: clipHeight },
   })
@@ -1580,7 +1647,7 @@ export async function screenshotWithAccessibilityLabels({
       return rawBuffer
     }
     try {
-      const result = await resizeImage({ input: rawBuffer })
+      const result = await resizeImageForAgent({ input: rawBuffer, format: 'png' })
       return result.buffer
     } catch (err) {
       logger?.error?.('[playwriter] sharp resize failed, using raw buffer:', err)
@@ -1604,7 +1671,7 @@ export async function screenshotWithAccessibilityLabels({
   collector.push({
     path: screenshotPath,
     base64,
-    mimeType: 'image/jpeg',
+    mimeType: 'image/png',
     snapshot,
     labelCount,
   })

package/src/aria-snapshots/contenteditable-interactive.txt

@@ -0,0 +1,7 @@
+      - labeltext:
+        - text: "Editor:"
+      - textbox "Main editor" [id="editor1"]
+      - role=textbox >> nth=0
+      - role=textbox >> nth=1
+    - role=textbox[name="Normal text input for comparison"]
+    - role=textbox[name="Normal textarea for comparison"]

package/src/aria-snapshots/contenteditable-raw.txt

@@ -0,0 +1,14 @@
+    - heading "Rich Text Editor Test"
+      - labeltext:
+        - text: "Editor:"
+      - textbox "Main editor" [id="editor1"]:
+        - paragraph:
+          - text: "This is some initial content in a contenteditable editor."
+      - role=textbox >> nth=0:
+        - paragraph:
+          - text: "Second editor with no label or id."
+      - role=textbox >> nth=1:
+        - paragraph:
+          - text: "Third editor simulating ProseMirror class."
+    - role=textbox[name="Normal text input for comparison"]
+    - role=textbox[name="Normal textarea for comparison"]

package/src/aria-snapshots/github-interactive.txt

@@ -1,6 +1,4 @@
       - role=link[name="Skip to content"]
-      - role=link[name="join the product roadmap webinar"]
-      - role=button[name="Close"]
       - banner:
           - role=link[name="Homepage"]
             - navigation "Global":

package/src/aria-snapshots/github-raw.txt

@@ -1,8 +1,4 @@
       - role=link[name="Skip to content"]
-      - alert "Announcement":
-        - text: "Inside GitHub’s roadmap for agentic development —"
-        - role=link[name="join the product roadmap webinar"]
-        - role=button[name="Close"]
       - banner:
         - heading "Navigation Menu"
           - role=link[name="Homepage"]