npm - playwriter - Versions diffs - 0.0.38 → 0.0.40 - Mend

playwriter 0.0.38 → 0.0.40

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (19) hide show

package/dist/aria-snapshot.d.ts +29 -2
package/dist/aria-snapshot.d.ts.map +1 -1
package/dist/aria-snapshot.js +103 -26
package/dist/aria-snapshot.js.map +1 -1
package/dist/mcp.d.ts.map +1 -1
package/dist/mcp.js +30 -13
package/dist/mcp.js.map +1 -1
package/dist/mcp.test.js +73 -0
package/dist/mcp.test.js.map +1 -1
package/package.json +13 -12
package/src/aria-snapshot.ts +129 -26
package/src/assets/aria-labels-github.png +0 -0
package/src/assets/aria-labels-google-snapshot.txt +1 -1
package/src/assets/aria-labels-google.png +0 -0
package/src/assets/aria-labels-hacker-news-snapshot.txt +826 -811
package/src/assets/aria-labels-hacker-news.png +0 -0
package/src/mcp.test.ts +86 -0
package/src/mcp.ts +35 -14
package/src/prompt.md +10 -11

package/src/assets/aria-labels-hacker-news.png CHANGED Viewed

Binary file

package/src/mcp.test.ts CHANGED Viewed

@@ -2307,6 +2307,76 @@ describe('MCP Server Tests', () => {
         console.log(`Screenshots saved to: ${assetsDir}`)
     }, 120000)
+    it('should take screenshot with accessibility labels via MCP execute tool', async () => {
+        const browserContext = getBrowserContext()
+        const serviceWorker = await getExtensionServiceWorker(browserContext)
+        const page = await browserContext.newPage()
+        await page.setContent(`
+            <html>
+                <body>
+                    <button id="submit-btn">Submit Form</button>
+                    <a href="/about">About Us</a>
+                    <input type="text" placeholder="Enter your name" />
+                </body>
+            </html>
+        `)
+        await page.bringToFront()
+        await serviceWorker.evaluate(async () => {
+            await globalThis.toggleExtensionForActiveTab()
+        })
+        await new Promise(r => setTimeout(r, 400))
+        // Take screenshot with accessibility labels via MCP
+        const result = await client.callTool({
+            name: 'execute',
+            arguments: {
+                code: js`
+                    let testPage;
+                    for (const p of context.pages()) {
+                        const html = await p.content();
+                        if (html.includes('submit-btn')) { testPage = p; break; }
+                    }
+                    if (!testPage) throw new Error('Test page not found');
+                    await screenshotWithAccessibilityLabels({ page: testPage });
+                `,
+                timeout: 15000,
+            },
+        })
+        expect(result.isError).toBeFalsy()
+        // Verify response has both text and image content
+        const content = result.content as any[]
+        expect(content.length).toBe(2)
+        // Check text content
+        const textContent = content.find(c => c.type === 'text')
+        expect(textContent).toBeDefined()
+        expect(textContent.text).toContain('Screenshot saved to:')
+        expect(textContent.text).toContain('.jpg')
+        expect(textContent.text).toContain('Labels shown:')
+        expect(textContent.text).toContain('Accessibility snapshot:')
+        expect(textContent.text).toContain('Submit Form')
+        // Check image content
+        const imageContent = content.find(c => c.type === 'image')
+        expect(imageContent).toBeDefined()
+        expect(imageContent.mimeType).toBe('image/jpeg')
+        expect(imageContent.data).toBeDefined()
+        expect(imageContent.data.length).toBeGreaterThan(100) // base64 data should be substantial
+        // Verify the image is valid JPEG by checking base64
+        const buffer = Buffer.from(imageContent.data, 'base64')
+        const dimensions = imageSize(buffer)
+        expect(dimensions.type).toBe('jpg')
+        expect(dimensions.width).toBeGreaterThan(0)
+        expect(dimensions.height).toBeGreaterThan(0)
+        await page.close()
+    }, 60000)
 })
@@ -2648,6 +2718,11 @@ describe('CDP Session Tests', () => {
         const browserContext = getBrowserContext()
         const serviceWorker = await getExtensionServiceWorker(browserContext)
+        // Clear any existing connected tabs from previous tests
+        await serviceWorker.evaluate(async () => {
+            await globalThis.disconnectEverything()
+        })
         const page = await browserContext.newPage()
         await page.goto('https://example.com/')
         await page.bringToFront()
@@ -2692,6 +2767,11 @@ describe('CDP Session Tests', () => {
         const browserContext = getBrowserContext()
         const serviceWorker = await getExtensionServiceWorker(browserContext)
+        // Clear any existing connected tabs from previous tests
+        await serviceWorker.evaluate(async () => {
+            await globalThis.disconnectEverything()
+        })
         const page1 = await browserContext.newPage()
         await page1.goto('https://example.com/')
         await page1.bringToFront()
@@ -3267,6 +3347,12 @@ describe('Auto-enable Tests', () => {
         const browserContext = getBrowserContext()
         const serviceWorker = await getExtensionServiceWorker(browserContext)
+        // Ensure clean state - disconnect any tabs from previous tests or setup
+        await serviceWorker.evaluate(async () => {
+            await globalThis.disconnectEverything()
+        })
+        await new Promise(r => setTimeout(r, 100))
         // Verify no tabs are connected
         const tabCountBefore = await serviceWorker.evaluate(() => {
             const state = globalThis.getExtensionState()

package/src/mcp.ts CHANGED Viewed

@@ -21,7 +21,7 @@ import { Editor } from './editor.js'
 import { getStylesForLocator, formatStylesAsText, type StylesResult } from './styles.js'
 import { getReactSource, type ReactSourceLocation } from './react-source.js'
 import { ScopedFS } from './scoped-fs.js'
-import { showAriaRefLabels, hideAriaRefLabels } from './aria-snapshot.js'
+import { screenshotWithAccessibilityLabels, type ScreenshotResult } from './aria-snapshot.js'
 const __filename = fileURLToPath(import.meta.url)
 const __dirname = path.dirname(__filename)
@@ -86,8 +86,7 @@ interface VMContext {
   getStylesForLocator: (options: { locator: any }) => Promise<StylesResult>
   formatStylesAsText: (styles: StylesResult) => string
   getReactSource: (options: { locator: any }) => Promise<ReactSourceLocation | null>
-  showAriaRefLabels: (options: { page: Page; interactiveOnly?: boolean }) => Promise<{ snapshot: string; labelCount: number }>
-  hideAriaRefLabels: (options: { page: Page }) => Promise<void>
+  screenshotWithAccessibilityLabels: (options: { page: Page; interactiveOnly?: boolean }) => Promise<void>
   require: NodeRequire
   import: (specifier: string) => Promise<any>
 }
@@ -864,6 +863,13 @@ server.tool(
         return getReactSource({ locator: options.locator, cdp })
       }
+      // Collector for screenshots taken during this execution
+      const screenshotCollector: ScreenshotResult[] = []
+      const screenshotWithAccessibilityLabelsFn = async (options: { page: Page; interactiveOnly?: boolean }) => {
+        return screenshotWithAccessibilityLabels({ ...options, collector: screenshotCollector })
+      }
       let vmContextObj: VMContextWithGlobals = {
         page,
         context,
@@ -880,8 +886,7 @@ server.tool(
         getStylesForLocator: getStylesForLocatorFn,
         formatStylesAsText,
         getReactSource: getReactSourceFn,
-        showAriaRefLabels,
-        hideAriaRefLabels,
+        screenshotWithAccessibilityLabels: screenshotWithAccessibilityLabelsFn,
         resetPlaywright: async () => {
           const { page: newPage, context: newContext } = await resetConnection()
@@ -901,8 +906,7 @@ server.tool(
             getStylesForLocator: getStylesForLocatorFn,
             formatStylesAsText,
             getReactSource: getReactSourceFn,
-            showAriaRefLabels,
-            hideAriaRefLabels,
+            screenshotWithAccessibilityLabels: screenshotWithAccessibilityLabelsFn,
             resetPlaywright: vmContextObj.resetPlaywright,
             require: sandboxedRequire,
             // TODO --experimental-vm-modules is needed to make import work in vm
@@ -943,6 +947,13 @@ server.tool(
         responseText += 'Code executed successfully (no output)'
       }
+      // Add screenshot info to response text
+      for (const screenshot of screenshotCollector) {
+        responseText += `\nScreenshot saved to: ${screenshot.path}\n`
+        responseText += `Labels shown: ${screenshot.labelCount}\n\n`
+        responseText += `Accessibility snapshot:\n${screenshot.snapshot}\n`
+      }
       const MAX_LENGTH = 6000
       let finalText = responseText.trim()
       if (finalText.length > MAX_LENGTH) {
@@ -951,14 +962,24 @@ server.tool(
           `\n\n[Truncated to ${MAX_LENGTH} characters. Better manage your logs or paginate them to read the full logs]`
       }
-      return {
-        content: [
-          {
-            type: 'text',
-            text: finalText,
-          },
-        ],
+      // Build content array with text and any collected screenshots
+      const content: Array<{ type: 'text'; text: string } | { type: 'image'; data: string; mimeType: string }> = [
+        {
+          type: 'text',
+          text: finalText,
+        },
+      ]
+      // Add all collected screenshots as images
+      for (const screenshot of screenshotCollector) {
+        content.push({
+          type: 'image',
+          data: screenshot.base64,
+          mimeType: screenshot.mimeType,
+        })
       }
+      return { content }
     } catch (error: any) {
       const errorStack = error.stack || error.message
       const isTimeoutError = error instanceof CodeExecutionTimeoutError || error.name === 'TimeoutError'

package/src/prompt.md CHANGED Viewed

@@ -206,24 +206,23 @@ const matches = await editor.grep({ regex: /console\.log/ });
 await editor.edit({ url: matches[0].url, oldString: 'DEBUG = false', newString: 'DEBUG = true' });
 ```
-**showAriaRefLabels** - overlay Vimium-style visual labels on interactive elements. Useful for taking screenshots where you can see element references. Labels auto-hide after 5 seconds. Call again if page HTML changes to get fresh labels.
+**screenshotWithAccessibilityLabels** - take a screenshot with Vimium-style visual labels overlaid on interactive elements. Shows labels, captures screenshot, then removes labels. The image and accessibility snapshot are automatically included in the response. Can be called multiple times to capture multiple screenshots. Use a timeout of 10 seconds at least.
 ```js
-const { snapshot, labelCount } = await showAriaRefLabels({ page });
-console.log(`Showing ${labelCount} labels`);
-await page.screenshot({ path: '/tmp/labeled-page.png' });
-// Use aria-ref from snapshot to interact
+await screenshotWithAccessibilityLabels({ page });
+// Image and accessibility snapshot are automatically included in response
+// Use aria-ref from snapshot to interact with elements
 await page.locator('aria-ref=e5').click();
+// Can take multiple screenshots in one execution
+await screenshotWithAccessibilityLabels({ page });
+await page.click('button');
+await screenshotWithAccessibilityLabels({ page });
+// Both images are included in the response
 ```
 Labels are color-coded: yellow=links, orange=buttons, coral=inputs, pink=checkboxes, peach=sliders, salmon=menus, amber=tabs.
-**hideAriaRefLabels** - manually remove labels before the 5-second auto-hide:
-```js
-await hideAriaRefLabels({ page });
-```
 ## pinned elements
 Users can right-click → "Copy Playwriter Element Reference" to store elements in `globalThis.playwriterPinnedElem1` (increments for each pin). The reference is copied to clipboard: