playwriter 0.0.89 → 0.0.103
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/aria-snapshot.d.ts +7 -3
- package/dist/aria-snapshot.d.ts.map +1 -1
- package/dist/aria-snapshot.js +76 -18
- package/dist/aria-snapshot.js.map +1 -1
- package/dist/aria-snapshot.test.js +94 -0
- package/dist/aria-snapshot.test.js.map +1 -1
- package/dist/bippy.js +1 -1
- package/dist/browser-config.d.ts +12 -1
- package/dist/browser-config.d.ts.map +1 -1
- package/dist/browser-config.js +101 -38
- package/dist/browser-config.js.map +1 -1
- package/dist/browser-launch.d.ts +17 -0
- package/dist/browser-launch.d.ts.map +1 -0
- package/dist/browser-launch.js +44 -0
- package/dist/browser-launch.js.map +1 -0
- package/dist/cdp-relay.d.ts.map +1 -1
- package/dist/cdp-relay.js +34 -10
- package/dist/cdp-relay.js.map +1 -1
- package/dist/channel-owner-inspect.test.d.ts +2 -0
- package/dist/channel-owner-inspect.test.d.ts.map +1 -0
- package/dist/channel-owner-inspect.test.js +75 -0
- package/dist/channel-owner-inspect.test.js.map +1 -0
- package/dist/chrome-discovery.d.ts +65 -0
- package/dist/chrome-discovery.d.ts.map +1 -0
- package/dist/chrome-discovery.js +165 -0
- package/dist/chrome-discovery.js.map +1 -0
- package/dist/chrome-discovery.test.d.ts +2 -0
- package/dist/chrome-discovery.test.d.ts.map +1 -0
- package/dist/chrome-discovery.test.js +41 -0
- package/dist/chrome-discovery.test.js.map +1 -0
- package/dist/cli.js +357 -53
- package/dist/cli.js.map +1 -1
- package/dist/executor.d.ts +44 -11
- package/dist/executor.d.ts.map +1 -1
- package/dist/executor.js +121 -55
- package/dist/executor.js.map +1 -1
- package/dist/executor.unit.test.js +29 -1
- package/dist/executor.unit.test.js.map +1 -1
- package/dist/extension/background.js +1704 -0
- package/dist/extension/icons/SolarCursorSquareBold.png +0 -0
- package/dist/extension/icons/icon-black-128.png +0 -0
- package/dist/extension/icons/icon-black-16.png +0 -0
- package/dist/extension/icons/icon-black-32.png +0 -0
- package/dist/extension/icons/icon-black-48.png +0 -0
- package/dist/extension/icons/icon-gray-128.png +0 -0
- package/dist/extension/icons/icon-gray-16.png +0 -0
- package/dist/extension/icons/icon-gray-32.png +0 -0
- package/dist/extension/icons/icon-gray-48.png +0 -0
- package/dist/extension/icons/icon-green-128.png +0 -0
- package/dist/extension/icons/icon-green-16.png +0 -0
- package/dist/extension/icons/icon-green-32.png +0 -0
- package/dist/extension/icons/icon-green-48.png +0 -0
- package/dist/extension/icons/playwriter-icon-black.png +0 -0
- package/dist/extension/icons/playwriter-icon-gray-disabled.png +0 -0
- package/dist/extension/icons/playwriter-icon-gray-disabled.svg +1 -0
- package/dist/extension/manifest.json +40 -0
- package/dist/extension/offscreen.js +214 -0
- package/dist/extension/src/offscreen.html +9 -0
- package/dist/extension/src/prism-bash.min.js +1 -0
- package/dist/extension/src/prism.min.js +1 -0
- package/dist/extension/src/welcome.html +399 -0
- package/dist/kitty-graphics.d.ts +22 -0
- package/dist/kitty-graphics.d.ts.map +1 -0
- package/dist/kitty-graphics.js +71 -0
- package/dist/kitty-graphics.js.map +1 -0
- package/dist/kitty-graphics.test.d.ts +2 -0
- package/dist/kitty-graphics.test.d.ts.map +1 -0
- package/dist/kitty-graphics.test.js +125 -0
- package/dist/kitty-graphics.test.js.map +1 -0
- package/dist/mcp.d.ts.map +1 -1
- package/dist/mcp.js +76 -12
- package/dist/mcp.js.map +1 -1
- package/dist/package-paths.d.ts +3 -0
- package/dist/package-paths.d.ts.map +1 -0
- package/dist/package-paths.js +29 -0
- package/dist/package-paths.js.map +1 -0
- package/dist/prompt.md +4 -2
- package/dist/readability.js +1 -1
- package/dist/relay-core.test.js +14 -4
- package/dist/relay-core.test.js.map +1 -1
- package/dist/relay-navigation.test.d.ts.map +1 -1
- package/dist/relay-navigation.test.js +95 -1
- package/dist/relay-navigation.test.js.map +1 -1
- package/dist/scoped-fs.d.ts +5 -4
- package/dist/scoped-fs.d.ts.map +1 -1
- package/dist/scoped-fs.js +11 -9
- package/dist/scoped-fs.js.map +1 -1
- package/dist/scoped-fs.test.d.ts +5 -0
- package/dist/scoped-fs.test.d.ts.map +1 -0
- package/dist/scoped-fs.test.js +70 -0
- package/dist/scoped-fs.test.js.map +1 -0
- package/dist/screen-recording.d.ts +3 -6
- package/dist/screen-recording.d.ts.map +1 -1
- package/dist/screen-recording.js +7 -21
- package/dist/screen-recording.js.map +1 -1
- package/dist/selector-generator.js +1 -1
- package/dist/snapshot-tools.test.js +3 -4
- package/dist/snapshot-tools.test.js.map +1 -1
- package/dist/test-utils.d.ts +3 -1
- package/dist/test-utils.d.ts.map +1 -1
- package/dist/test-utils.js +24 -12
- package/dist/test-utils.js.map +1 -1
- package/package.json +6 -5
- package/src/aria-snapshot.test.ts +110 -0
- package/src/aria-snapshot.ts +86 -19
- package/src/aria-snapshots/contenteditable-interactive.txt +7 -0
- package/src/aria-snapshots/contenteditable-raw.txt +14 -0
- package/src/aria-snapshots/github-interactive.txt +0 -2
- package/src/aria-snapshots/github-raw.txt +0 -4
- package/src/aria-snapshots/hackernews-interactive.txt +240 -237
- package/src/aria-snapshots/hackernews-raw.txt +262 -258
- package/src/aria-snapshots/prosemirror-interactive.txt +63 -0
- package/src/aria-snapshots/prosemirror-raw.txt +141 -0
- package/src/assets/aria-labels-hacker-news.png +0 -0
- package/src/browser-config.ts +143 -37
- package/src/browser-launch.ts +75 -0
- package/src/cdp-relay.ts +46 -11
- package/src/channel-owner-inspect.test.ts +94 -0
- package/src/chrome-discovery.test.ts +50 -0
- package/src/chrome-discovery.ts +211 -0
- package/src/cli.ts +432 -59
- package/src/executor.ts +171 -72
- package/src/executor.unit.test.ts +32 -1
- package/src/kitty-graphics.test.ts +139 -0
- package/src/kitty-graphics.ts +79 -0
- package/src/mcp.ts +82 -12
- package/src/package-paths.ts +36 -0
- package/src/relay-core.test.ts +14 -4
- package/src/relay-navigation.test.ts +106 -1
- package/src/scoped-fs.test.ts +77 -0
- package/src/scoped-fs.ts +12 -9
- package/src/screen-recording.ts +7 -22
- package/src/skill.md +32 -2
- package/src/snapshot-tools.test.ts +3 -4
- package/src/snapshots/shadcn-ui-accessibility-full.md +181 -183
- package/src/snapshots/shadcn-ui-accessibility-interactive.md +119 -121
- package/src/test-utils.ts +25 -11
package/src/executor.ts
CHANGED
|
@@ -26,7 +26,7 @@ import { ScopedFS } from './scoped-fs.js'
|
|
|
26
26
|
import {
|
|
27
27
|
screenshotWithAccessibilityLabels,
|
|
28
28
|
getAriaSnapshot,
|
|
29
|
-
|
|
29
|
+
resizeImageForAgent,
|
|
30
30
|
type ScreenshotResult,
|
|
31
31
|
type SnapshotFormat,
|
|
32
32
|
} from './aria-snapshot.js'
|
|
@@ -192,9 +192,18 @@ const ALLOWED_MODULES = new Set([
|
|
|
192
192
|
'node:fs',
|
|
193
193
|
])
|
|
194
194
|
|
|
195
|
+
export interface ExecuteScreenshot {
|
|
196
|
+
path: string
|
|
197
|
+
base64: string
|
|
198
|
+
mimeType: 'image/png'
|
|
199
|
+
snapshot: string
|
|
200
|
+
labelCount: number
|
|
201
|
+
}
|
|
202
|
+
|
|
195
203
|
export interface ExecuteResult {
|
|
196
204
|
text: string
|
|
197
205
|
images: Array<{ data: string; mimeType: string }>
|
|
206
|
+
screenshots: ExecuteScreenshot[]
|
|
198
207
|
isError: boolean
|
|
199
208
|
}
|
|
200
209
|
|
|
@@ -217,6 +226,8 @@ export interface CdpConfig {
|
|
|
217
226
|
port?: number
|
|
218
227
|
token?: string
|
|
219
228
|
extensionId?: string | null
|
|
229
|
+
/** Direct CDP WebSocket URL — bypasses relay + extension, connects straight to Chrome */
|
|
230
|
+
directCdpUrl?: string
|
|
220
231
|
}
|
|
221
232
|
|
|
222
233
|
export interface SessionMetadata {
|
|
@@ -225,6 +236,15 @@ export interface SessionMetadata {
|
|
|
225
236
|
profile: { email: string; id: string } | null
|
|
226
237
|
}
|
|
227
238
|
|
|
239
|
+
export interface SessionInfo {
|
|
240
|
+
id: string
|
|
241
|
+
stateKeys: string[]
|
|
242
|
+
extensionId: string | null
|
|
243
|
+
browser: string | null
|
|
244
|
+
profile: { email: string; id: string } | null
|
|
245
|
+
cwd: string | null
|
|
246
|
+
}
|
|
247
|
+
|
|
228
248
|
export interface ExecutorOptions {
|
|
229
249
|
cdpConfig: CdpConfig
|
|
230
250
|
sessionMetadata?: SessionMetadata
|
|
@@ -243,6 +263,23 @@ function isPromise(value: any): value is Promise<unknown> {
|
|
|
243
263
|
return typeof value === 'object' && value !== null && typeof value.then === 'function'
|
|
244
264
|
}
|
|
245
265
|
|
|
266
|
+
/**
|
|
267
|
+
* Duck-type check for a Playwright ChannelOwner (Response, Page, Browser,
|
|
268
|
+
* Request, Frame, BrowserContext, etc.). Used to skip auto-printing these
|
|
269
|
+
* objects from the REPL — they're meant for programmatic use, and dumping
|
|
270
|
+
* them risks leaking internal fields. Users can still `console.log(obj)` to
|
|
271
|
+
* inspect them via the safe handler in playwright-core. See issue #82.
|
|
272
|
+
*/
|
|
273
|
+
export function isPlaywrightChannelOwner(value: any): boolean {
|
|
274
|
+
return (
|
|
275
|
+
value !== null &&
|
|
276
|
+
typeof value === 'object' &&
|
|
277
|
+
typeof value._type === 'string' &&
|
|
278
|
+
typeof value._guid === 'string' &&
|
|
279
|
+
value._connection !== undefined
|
|
280
|
+
)
|
|
281
|
+
}
|
|
282
|
+
|
|
246
283
|
export class PlaywrightExecutor {
|
|
247
284
|
private isConnected = false
|
|
248
285
|
private page: Page | null = null
|
|
@@ -272,14 +309,19 @@ export class PlaywrightExecutor {
|
|
|
272
309
|
private cdpConfig: CdpConfig
|
|
273
310
|
private logger: ExecutorLogger
|
|
274
311
|
private sessionMetadata: SessionMetadata
|
|
312
|
+
private sessionCwd: string | null
|
|
275
313
|
private hasWarnedExtensionOutdated = false
|
|
276
314
|
|
|
277
315
|
constructor(options: ExecutorOptions) {
|
|
278
316
|
this.cdpConfig = options.cdpConfig
|
|
279
317
|
this.logger = options.logger || { log: console.log, error: console.error }
|
|
280
318
|
this.sessionMetadata = options.sessionMetadata || { extensionId: null, browser: null, profile: null }
|
|
319
|
+
this.sessionCwd = options.cwd ? path.resolve(options.cwd) : null
|
|
281
320
|
// ScopedFS expects an array of allowed directories. If cwd is provided, use it; otherwise use defaults.
|
|
282
|
-
this.scopedFs = new ScopedFS(
|
|
321
|
+
this.scopedFs = new ScopedFS(
|
|
322
|
+
this.sessionCwd ? [this.sessionCwd, '/tmp', os.tmpdir()] : undefined,
|
|
323
|
+
this.sessionCwd || undefined,
|
|
324
|
+
)
|
|
283
325
|
this.sandboxedRequire = this.createSandboxedRequire(require)
|
|
284
326
|
}
|
|
285
327
|
|
|
@@ -568,19 +610,55 @@ export class PlaywrightExecutor {
|
|
|
568
610
|
}
|
|
569
611
|
}
|
|
570
612
|
|
|
571
|
-
private
|
|
572
|
-
|
|
573
|
-
|
|
613
|
+
private isDirectCdpMode(): boolean {
|
|
614
|
+
return !!this.cdpConfig.directCdpUrl
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
/**
|
|
618
|
+
* Connect to Chrome and set up context/page. Shared by ensureConnection and reset.
|
|
619
|
+
* In direct CDP mode, connects straight to Chrome's WebSocket.
|
|
620
|
+
* In extension mode, checks extension status then connects via relay.
|
|
621
|
+
*/
|
|
622
|
+
private async connectToBrowser(): Promise<{ browser: Browser; page: Page; context: BrowserContext }> {
|
|
623
|
+
if (this.isDirectCdpMode()) {
|
|
624
|
+
// Direct CDP: connect straight to Chrome, no relay or extension needed
|
|
625
|
+
const browser = await chromium.connectOverCDP(this.cdpConfig.directCdpUrl!)
|
|
626
|
+
|
|
627
|
+
browser.on('disconnected', () => {
|
|
628
|
+
this.logger.log('Browser disconnected, clearing connection state')
|
|
629
|
+
this.clearConnectionState()
|
|
630
|
+
})
|
|
631
|
+
|
|
632
|
+
const contexts = browser.contexts()
|
|
633
|
+
const context = contexts.length > 0 ? contexts[0] : await browser.newContext()
|
|
634
|
+
|
|
635
|
+
context.setDefaultTimeout(60000)
|
|
636
|
+
context.setDefaultNavigationTimeout(10000)
|
|
637
|
+
|
|
638
|
+
context.on('page', (page) => {
|
|
639
|
+
this.setupPageListeners(page)
|
|
640
|
+
})
|
|
641
|
+
|
|
642
|
+
context.pages().forEach((p) => this.setupPageListeners(p))
|
|
643
|
+
|
|
644
|
+
// In direct CDP mode, pages are always available (all tabs visible).
|
|
645
|
+
// Use the first non-closed page, or create one.
|
|
646
|
+
const pages = context.pages().filter((p) => !p.isClosed())
|
|
647
|
+
const page = pages.length > 0 ? pages[0] : await context.newPage()
|
|
648
|
+
this.setupPageListeners(page)
|
|
649
|
+
|
|
650
|
+
await this.setDeviceScaleFactorForMacOS(context)
|
|
651
|
+
|
|
652
|
+
return { browser, page, context }
|
|
574
653
|
}
|
|
575
654
|
|
|
576
|
-
//
|
|
655
|
+
// Extension mode: check status first for better error messages
|
|
577
656
|
const extensionStatus = await this.checkExtensionStatus()
|
|
578
657
|
if (!extensionStatus.connected) {
|
|
579
658
|
throw new Error(EXTENSION_NOT_CONNECTED_ERROR)
|
|
580
659
|
}
|
|
581
660
|
this.warnIfExtensionOutdated(extensionStatus.playwriterVersion)
|
|
582
661
|
|
|
583
|
-
// Generate a fresh unique URL for each Playwright connection
|
|
584
662
|
const cdpUrl = getCdpUrl(this.cdpConfig)
|
|
585
663
|
const browser = await chromium.connectOverCDP(cdpUrl)
|
|
586
664
|
|
|
@@ -607,6 +685,16 @@ export class PlaywrightExecutor {
|
|
|
607
685
|
|
|
608
686
|
await this.setDeviceScaleFactorForMacOS(context)
|
|
609
687
|
|
|
688
|
+
return { browser, page, context }
|
|
689
|
+
}
|
|
690
|
+
|
|
691
|
+
private async ensureConnection(): Promise<{ browser: Browser; page: Page }> {
|
|
692
|
+
if (this.isConnected && this.browser && this.page) {
|
|
693
|
+
return { browser: this.browser, page: this.page }
|
|
694
|
+
}
|
|
695
|
+
|
|
696
|
+
const { browser, page, context } = await this.connectToBrowser()
|
|
697
|
+
|
|
610
698
|
this.browser = browser
|
|
611
699
|
this.page = page
|
|
612
700
|
this.context = context
|
|
@@ -656,39 +744,7 @@ export class PlaywrightExecutor {
|
|
|
656
744
|
this.clearConnectionState()
|
|
657
745
|
this.clearUserState()
|
|
658
746
|
|
|
659
|
-
|
|
660
|
-
const extensionStatus = await this.checkExtensionStatus()
|
|
661
|
-
if (!extensionStatus.connected) {
|
|
662
|
-
throw new Error(EXTENSION_NOT_CONNECTED_ERROR)
|
|
663
|
-
}
|
|
664
|
-
this.warnIfExtensionOutdated(extensionStatus.playwriterVersion)
|
|
665
|
-
|
|
666
|
-
// Generate a fresh unique URL for each Playwright connection
|
|
667
|
-
const cdpUrl = getCdpUrl(this.cdpConfig)
|
|
668
|
-
const browser = await chromium.connectOverCDP(cdpUrl)
|
|
669
|
-
|
|
670
|
-
browser.on('disconnected', () => {
|
|
671
|
-
this.logger.log('Browser disconnected, clearing connection state')
|
|
672
|
-
this.clearConnectionState()
|
|
673
|
-
})
|
|
674
|
-
|
|
675
|
-
const contexts = browser.contexts()
|
|
676
|
-
const context = contexts.length > 0 ? contexts[0] : await browser.newContext()
|
|
677
|
-
|
|
678
|
-
// Action timeout (click, fill, hover, etc.) is longer to tolerate slower
|
|
679
|
-
// SPA/Turbo navigations and post-click settling on real sites.
|
|
680
|
-
// Navigation timeout (goto, reload) remains separate.
|
|
681
|
-
context.setDefaultTimeout(60000)
|
|
682
|
-
context.setDefaultNavigationTimeout(10000)
|
|
683
|
-
|
|
684
|
-
context.on('page', (page) => {
|
|
685
|
-
this.setupPageListeners(page)
|
|
686
|
-
})
|
|
687
|
-
|
|
688
|
-
context.pages().forEach((p) => this.setupPageListeners(p))
|
|
689
|
-
const page = await this.ensurePageForContext({ context, timeout: 10000 })
|
|
690
|
-
|
|
691
|
-
await this.setDeviceScaleFactorForMacOS(context)
|
|
747
|
+
const { browser, page, context } = await this.connectToBrowser()
|
|
692
748
|
|
|
693
749
|
this.browser = browser
|
|
694
750
|
this.page = page
|
|
@@ -971,6 +1027,15 @@ export class PlaywrightExecutor {
|
|
|
971
1027
|
}
|
|
972
1028
|
|
|
973
1029
|
const screenshotCollector: ScreenshotResult[] = []
|
|
1030
|
+
// Separate collector for images produced by resizeImageForAgent() calls.
|
|
1031
|
+
// These get merged into result.images so the CLI can emit them via Kitty Graphics.
|
|
1032
|
+
const resizedImageCollector: Array<{ data: string; mimeType: string }> = []
|
|
1033
|
+
|
|
1034
|
+
const resizeImageForAgentFn: typeof resizeImageForAgent = async (options) => {
|
|
1035
|
+
const result = await resizeImageForAgent(options)
|
|
1036
|
+
resizedImageCollector.push({ data: result.buffer.toString('base64'), mimeType: result.mimeType })
|
|
1037
|
+
return result
|
|
1038
|
+
}
|
|
974
1039
|
|
|
975
1040
|
const screenshotWithAccessibilityLabelsFn = async (options: { page: Page; interactiveOnly?: boolean }) => {
|
|
976
1041
|
return screenshotWithAccessibilityLabels({
|
|
@@ -1052,6 +1117,7 @@ export class PlaywrightExecutor {
|
|
|
1052
1117
|
let vmContextObj: any = {
|
|
1053
1118
|
page,
|
|
1054
1119
|
context,
|
|
1120
|
+
browser: this.browser,
|
|
1055
1121
|
state: this.userState,
|
|
1056
1122
|
console: customConsole,
|
|
1057
1123
|
snapshot,
|
|
@@ -1070,7 +1136,9 @@ export class PlaywrightExecutor {
|
|
|
1070
1136
|
formatStylesAsText,
|
|
1071
1137
|
getReactSource: getReactSourceFn,
|
|
1072
1138
|
screenshotWithAccessibilityLabels: screenshotWithAccessibilityLabelsFn,
|
|
1073
|
-
|
|
1139
|
+
resizeImageForAgent: resizeImageForAgentFn,
|
|
1140
|
+
// Backward-compatible alias for resizeImageForAgent
|
|
1141
|
+
resizeImage: resizeImageForAgentFn,
|
|
1074
1142
|
ghostCursor: {
|
|
1075
1143
|
show: showGhostCursor,
|
|
1076
1144
|
hide: hideGhostCursor,
|
|
@@ -1091,6 +1159,7 @@ export class PlaywrightExecutor {
|
|
|
1091
1159
|
const { page: newPage, context: newContext } = await self.reset()
|
|
1092
1160
|
vmContextObj.page = newPage
|
|
1093
1161
|
vmContextObj.context = newContext
|
|
1162
|
+
vmContextObj.browser = self.browser
|
|
1094
1163
|
return { page: newPage, context: newContext }
|
|
1095
1164
|
},
|
|
1096
1165
|
require: this.sandboxedRequire,
|
|
@@ -1138,7 +1207,12 @@ export class PlaywrightExecutor {
|
|
|
1138
1207
|
// Only show return value if user explicitly used return
|
|
1139
1208
|
if (hasExplicitReturn) {
|
|
1140
1209
|
const resolvedResult = isPromise(result) ? await result : result
|
|
1141
|
-
|
|
1210
|
+
// Auto-returned Playwright handles (Response, Page, Browser, Request,
|
|
1211
|
+
// Frame, etc.) are silently skipped — they're programmatic references,
|
|
1212
|
+
// not useful display data. Users can `console.log(response)` or
|
|
1213
|
+
// return specific fields (`return response.url()`) to see values.
|
|
1214
|
+
// See issue #82.
|
|
1215
|
+
if (resolvedResult !== undefined && !isPlaywrightChannelOwner(resolvedResult)) {
|
|
1142
1216
|
const formatted =
|
|
1143
1217
|
typeof resolvedResult === 'string'
|
|
1144
1218
|
? resolvedResult
|
|
@@ -1161,12 +1235,6 @@ export class PlaywrightExecutor {
|
|
|
1161
1235
|
responseText = 'Code executed successfully (no output)'
|
|
1162
1236
|
}
|
|
1163
1237
|
|
|
1164
|
-
for (const screenshot of screenshotCollector) {
|
|
1165
|
-
responseText += `\nScreenshot saved to: ${screenshot.path}\n`
|
|
1166
|
-
responseText += `Labels shown: ${screenshot.labelCount}\n\n`
|
|
1167
|
-
responseText += `Accessibility snapshot:\n${screenshot.snapshot}\n`
|
|
1168
|
-
}
|
|
1169
|
-
|
|
1170
1238
|
const MAX_LENGTH = 10000
|
|
1171
1239
|
let finalText = responseText.trim()
|
|
1172
1240
|
if (finalText.length > MAX_LENGTH) {
|
|
@@ -1175,9 +1243,19 @@ export class PlaywrightExecutor {
|
|
|
1175
1243
|
`\n\n[Truncated to ${MAX_LENGTH} characters. Use search to find specific content]`
|
|
1176
1244
|
}
|
|
1177
1245
|
|
|
1178
|
-
const images =
|
|
1179
|
-
|
|
1180
|
-
|
|
1246
|
+
const images = [
|
|
1247
|
+
...screenshotCollector.map((s) => ({ data: s.base64, mimeType: s.mimeType })),
|
|
1248
|
+
...resizedImageCollector,
|
|
1249
|
+
]
|
|
1250
|
+
const screenshots: ExecuteScreenshot[] = screenshotCollector.map((s) => ({
|
|
1251
|
+
path: s.path,
|
|
1252
|
+
base64: s.base64,
|
|
1253
|
+
mimeType: s.mimeType,
|
|
1254
|
+
snapshot: s.snapshot,
|
|
1255
|
+
labelCount: s.labelCount,
|
|
1256
|
+
}))
|
|
1257
|
+
|
|
1258
|
+
return { text: finalText, images, screenshots, isError: false }
|
|
1181
1259
|
} catch (error: any) {
|
|
1182
1260
|
const errorStack = error.stack || error.message
|
|
1183
1261
|
const isTimeoutError =
|
|
@@ -1196,12 +1274,14 @@ export class PlaywrightExecutor {
|
|
|
1196
1274
|
return {
|
|
1197
1275
|
text: `${logsText}${warningText}\nError executing code: ${errorText}${resetHint}`,
|
|
1198
1276
|
images: [],
|
|
1277
|
+
screenshots: [],
|
|
1199
1278
|
isError: true,
|
|
1200
1279
|
}
|
|
1201
1280
|
}
|
|
1202
1281
|
}
|
|
1203
1282
|
|
|
1204
1283
|
// When extension is connected but has no pages, auto-create only if PLAYWRITER_AUTO_ENABLE is set.
|
|
1284
|
+
// In direct CDP mode, always create a page (no extension check needed).
|
|
1205
1285
|
private async ensurePageForContext(options: { context: BrowserContext; timeout: number }): Promise<Page> {
|
|
1206
1286
|
const { context, timeout } = options
|
|
1207
1287
|
const pages = context.pages().filter((p) => !p.isClosed())
|
|
@@ -1209,6 +1289,14 @@ export class PlaywrightExecutor {
|
|
|
1209
1289
|
return pages[0]
|
|
1210
1290
|
}
|
|
1211
1291
|
|
|
1292
|
+
// Direct CDP mode: always create a new page, no extension involved
|
|
1293
|
+
if (this.isDirectCdpMode()) {
|
|
1294
|
+
const page = await context.newPage()
|
|
1295
|
+
this.setupPageListeners(page)
|
|
1296
|
+
await page.waitForLoadState('domcontentloaded', { timeout }).catch(() => {})
|
|
1297
|
+
return page
|
|
1298
|
+
}
|
|
1299
|
+
|
|
1212
1300
|
const extensionStatus = await this.checkExtensionStatus()
|
|
1213
1301
|
if (!extensionStatus.connected) {
|
|
1214
1302
|
throw new Error(EXTENSION_NOT_CONNECTED_ERROR)
|
|
@@ -1256,10 +1344,21 @@ export class PlaywrightExecutor {
|
|
|
1256
1344
|
getSessionMetadata(): SessionMetadata {
|
|
1257
1345
|
return this.sessionMetadata
|
|
1258
1346
|
}
|
|
1347
|
+
|
|
1348
|
+
getSessionInfo({ id }: { id: string }): SessionInfo {
|
|
1349
|
+
return {
|
|
1350
|
+
id,
|
|
1351
|
+
stateKeys: this.getStateKeys(),
|
|
1352
|
+
extensionId: this.sessionMetadata.extensionId,
|
|
1353
|
+
browser: this.sessionMetadata.browser,
|
|
1354
|
+
profile: this.sessionMetadata.profile,
|
|
1355
|
+
cwd: this.sessionCwd,
|
|
1356
|
+
}
|
|
1357
|
+
}
|
|
1259
1358
|
}
|
|
1260
1359
|
|
|
1261
1360
|
/**
|
|
1262
|
-
* Session manager for multiple executors, keyed by session ID
|
|
1361
|
+
* Session manager for multiple executors, keyed by session ID.
|
|
1263
1362
|
*/
|
|
1264
1363
|
export class ExecutorManager {
|
|
1265
1364
|
private executors = new Map<string, PlaywrightExecutor>()
|
|
@@ -1271,14 +1370,27 @@ export class ExecutorManager {
|
|
|
1271
1370
|
this.logger = options.logger || { log: console.log, error: console.error }
|
|
1272
1371
|
}
|
|
1273
1372
|
|
|
1274
|
-
getExecutor(options: {
|
|
1373
|
+
getExecutor(options: {
|
|
1374
|
+
sessionId: string
|
|
1375
|
+
cwd?: string
|
|
1376
|
+
sessionMetadata?: SessionMetadata
|
|
1377
|
+
/** Override cdpConfig for this session (e.g. direct CDP connection) */
|
|
1378
|
+
cdpConfig?: CdpConfig
|
|
1379
|
+
}): PlaywrightExecutor {
|
|
1275
1380
|
const { sessionId, cwd, sessionMetadata } = options
|
|
1276
1381
|
let executor = this.executors.get(sessionId)
|
|
1277
1382
|
if (!executor) {
|
|
1278
|
-
const
|
|
1279
|
-
|
|
1280
|
-
|
|
1281
|
-
|
|
1383
|
+
const cdpConfig = (() => {
|
|
1384
|
+
// Per-session override takes priority (used for direct CDP sessions)
|
|
1385
|
+
if (options.cdpConfig) {
|
|
1386
|
+
return options.cdpConfig
|
|
1387
|
+
}
|
|
1388
|
+
const baseConfig = typeof this.cdpConfig === 'function' ? this.cdpConfig(sessionId) : this.cdpConfig
|
|
1389
|
+
if (sessionMetadata?.extensionId) {
|
|
1390
|
+
return { ...baseConfig, extensionId: sessionMetadata.extensionId }
|
|
1391
|
+
}
|
|
1392
|
+
return baseConfig
|
|
1393
|
+
})()
|
|
1282
1394
|
executor = new PlaywrightExecutor({
|
|
1283
1395
|
cdpConfig,
|
|
1284
1396
|
sessionMetadata,
|
|
@@ -1298,22 +1410,9 @@ export class ExecutorManager {
|
|
|
1298
1410
|
return this.executors.get(sessionId) || null
|
|
1299
1411
|
}
|
|
1300
1412
|
|
|
1301
|
-
listSessions():
|
|
1302
|
-
id: string
|
|
1303
|
-
stateKeys: string[]
|
|
1304
|
-
extensionId: string | null
|
|
1305
|
-
browser: string | null
|
|
1306
|
-
profile: { email: string; id: string } | null
|
|
1307
|
-
}> {
|
|
1413
|
+
listSessions(): SessionInfo[] {
|
|
1308
1414
|
return [...this.executors.entries()].map(([id, executor]) => {
|
|
1309
|
-
|
|
1310
|
-
return {
|
|
1311
|
-
id,
|
|
1312
|
-
stateKeys: executor.getStateKeys(),
|
|
1313
|
-
extensionId: metadata.extensionId,
|
|
1314
|
-
browser: metadata.browser,
|
|
1315
|
-
profile: metadata.profile,
|
|
1316
|
-
}
|
|
1415
|
+
return executor.getSessionInfo({ id })
|
|
1317
1416
|
})
|
|
1318
1417
|
}
|
|
1319
1418
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { describe, it, expect } from 'vitest'
|
|
2
|
-
import { shouldAutoReturn, wrapCode } from './executor.js'
|
|
2
|
+
import { shouldAutoReturn, wrapCode, isPlaywrightChannelOwner } from './executor.js'
|
|
3
3
|
|
|
4
4
|
describe('shouldAutoReturn', () => {
|
|
5
5
|
it('returns true for simple expressions', () => {
|
|
@@ -135,3 +135,34 @@ describe('wrapCode', () => {
|
|
|
135
135
|
expect(wrapCode('x = 5')).toBe('(async () => { x = 5 })()')
|
|
136
136
|
})
|
|
137
137
|
})
|
|
138
|
+
|
|
139
|
+
describe('isPlaywrightChannelOwner', () => {
|
|
140
|
+
it('detects ChannelOwner-shaped objects', () => {
|
|
141
|
+
const fakeResponse = {
|
|
142
|
+
_type: 'Response',
|
|
143
|
+
_guid: 'response@abc123',
|
|
144
|
+
_connection: { _platform: { env: {} } },
|
|
145
|
+
}
|
|
146
|
+
expect(isPlaywrightChannelOwner(fakeResponse)).toBe(true)
|
|
147
|
+
})
|
|
148
|
+
|
|
149
|
+
it('rejects plain objects and primitives', () => {
|
|
150
|
+
expect(isPlaywrightChannelOwner(null)).toBe(false)
|
|
151
|
+
expect(isPlaywrightChannelOwner(undefined)).toBe(false)
|
|
152
|
+
expect(isPlaywrightChannelOwner('string')).toBe(false)
|
|
153
|
+
expect(isPlaywrightChannelOwner(42)).toBe(false)
|
|
154
|
+
expect(isPlaywrightChannelOwner({})).toBe(false)
|
|
155
|
+
expect(isPlaywrightChannelOwner([])).toBe(false)
|
|
156
|
+
expect(isPlaywrightChannelOwner({ url: 'https://example.com', status: 200 })).toBe(false)
|
|
157
|
+
})
|
|
158
|
+
|
|
159
|
+
it('rejects partial matches missing required fields', () => {
|
|
160
|
+
expect(isPlaywrightChannelOwner({ _type: 'Response' })).toBe(false)
|
|
161
|
+
expect(isPlaywrightChannelOwner({ _guid: 'abc' })).toBe(false)
|
|
162
|
+
expect(isPlaywrightChannelOwner({ _type: 'x', _guid: 'y' })).toBe(false)
|
|
163
|
+
// Missing _connection
|
|
164
|
+
expect(isPlaywrightChannelOwner({ _type: 'x', _guid: 'y' })).toBe(false)
|
|
165
|
+
// _type must be a string
|
|
166
|
+
expect(isPlaywrightChannelOwner({ _type: 123, _guid: 'y', _connection: {} })).toBe(false)
|
|
167
|
+
})
|
|
168
|
+
})
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
import { describe, expect, test } from 'vitest'
|
|
2
|
+
import { buildKittySequence, canEmitKittyGraphics } from './kitty-graphics.js'
|
|
3
|
+
|
|
4
|
+
// Minimal valid PNG (1x1 transparent pixel) as base64
|
|
5
|
+
const TINY_PNG_B64 =
|
|
6
|
+
'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=='
|
|
7
|
+
|
|
8
|
+
describe('kitty-graphics', () => {
|
|
9
|
+
describe('canEmitKittyGraphics', () => {
|
|
10
|
+
test('returns false when AGENT_GRAPHICS is not set', () => {
|
|
11
|
+
const prev = process.env.AGENT_GRAPHICS
|
|
12
|
+
delete process.env.AGENT_GRAPHICS
|
|
13
|
+
expect(canEmitKittyGraphics()).toBe(false)
|
|
14
|
+
if (prev !== undefined) {
|
|
15
|
+
process.env.AGENT_GRAPHICS = prev
|
|
16
|
+
}
|
|
17
|
+
})
|
|
18
|
+
|
|
19
|
+
test('returns true when AGENT_GRAPHICS=kitty', () => {
|
|
20
|
+
const prev = process.env.AGENT_GRAPHICS
|
|
21
|
+
process.env.AGENT_GRAPHICS = 'kitty'
|
|
22
|
+
expect(canEmitKittyGraphics()).toBe(true)
|
|
23
|
+
if (prev !== undefined) {
|
|
24
|
+
process.env.AGENT_GRAPHICS = prev
|
|
25
|
+
} else {
|
|
26
|
+
delete process.env.AGENT_GRAPHICS
|
|
27
|
+
}
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
test('returns true when AGENT_GRAPHICS contains kitty among others', () => {
|
|
31
|
+
const prev = process.env.AGENT_GRAPHICS
|
|
32
|
+
process.env.AGENT_GRAPHICS = 'kitty,iterm2'
|
|
33
|
+
expect(canEmitKittyGraphics()).toBe(true)
|
|
34
|
+
if (prev !== undefined) {
|
|
35
|
+
process.env.AGENT_GRAPHICS = prev
|
|
36
|
+
} else {
|
|
37
|
+
delete process.env.AGENT_GRAPHICS
|
|
38
|
+
}
|
|
39
|
+
})
|
|
40
|
+
})
|
|
41
|
+
|
|
42
|
+
describe('buildKittySequence', () => {
|
|
43
|
+
test('small payload produces single escape sequence', () => {
|
|
44
|
+
const result = buildKittySequence({ base64: 'AAAA' })
|
|
45
|
+
expect(result).toMatchInlineSnapshot(`"_Ga=T,f=100;AAAA\\"`)
|
|
46
|
+
})
|
|
47
|
+
|
|
48
|
+
test('sequence starts with APC and ends with ST', () => {
|
|
49
|
+
const result = buildKittySequence({ base64: 'AAAA' })
|
|
50
|
+
expect(result.startsWith('\x1b_G')).toBe(true)
|
|
51
|
+
expect(result.endsWith('\x1b\\')).toBe(true)
|
|
52
|
+
})
|
|
53
|
+
|
|
54
|
+
test('real PNG base64 produces valid sequence', () => {
|
|
55
|
+
const result = buildKittySequence({ base64: TINY_PNG_B64 })
|
|
56
|
+
// Should be a single chunk (small payload)
|
|
57
|
+
expect(result).toContain('a=T,f=100')
|
|
58
|
+
expect(result).toContain(TINY_PNG_B64)
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
test('large payload is chunked at 4096 bytes', () => {
|
|
62
|
+
// Create a payload larger than 4096 bytes
|
|
63
|
+
const bigPayload = 'A'.repeat(10000)
|
|
64
|
+
const result = buildKittySequence({ base64: bigPayload })
|
|
65
|
+
|
|
66
|
+
// Should have multiple escape sequences
|
|
67
|
+
// First chunk: a=T,f=100,m=1
|
|
68
|
+
expect(result).toContain('a=T,f=100,m=1')
|
|
69
|
+
// Last chunk: m=0
|
|
70
|
+
expect(result).toContain('\x1b_Gm=0;')
|
|
71
|
+
// Should NOT contain single-shot a=T,f=100; (without m=)
|
|
72
|
+
expect(result).not.toMatch(/a=T,f=100;/)
|
|
73
|
+
})
|
|
74
|
+
|
|
75
|
+
test('three-chunk payload has correct structure', () => {
|
|
76
|
+
// 9000 bytes = 3 chunks: 4096 + 4096 + 808
|
|
77
|
+
const payload = 'B'.repeat(9000)
|
|
78
|
+
const result = buildKittySequence({ base64: payload })
|
|
79
|
+
|
|
80
|
+
const sequences = result.split('\x1b\\').filter(Boolean)
|
|
81
|
+
expect(sequences.length).toBe(3)
|
|
82
|
+
|
|
83
|
+
// First: a=T,f=100,m=1
|
|
84
|
+
expect(sequences[0]).toContain('a=T,f=100,m=1')
|
|
85
|
+
// Middle: m=1
|
|
86
|
+
expect(sequences[1]).toContain('\x1b_Gm=1;')
|
|
87
|
+
// Last: m=0
|
|
88
|
+
expect(sequences[2]).toContain('\x1b_Gm=0;')
|
|
89
|
+
})
|
|
90
|
+
})
|
|
91
|
+
|
|
92
|
+
describe('round-trip with kitty-graphics-agent parser', () => {
|
|
93
|
+
test('emitted sequence can be parsed back to extract the image', async () => {
|
|
94
|
+
// Dynamically import the parser to test round-trip
|
|
95
|
+
let extractKittyGraphics: typeof import('kitty-graphics-agent/parser').extractKittyGraphics
|
|
96
|
+
try {
|
|
97
|
+
const mod = await import('kitty-graphics-agent/parser')
|
|
98
|
+
extractKittyGraphics = mod.extractKittyGraphics
|
|
99
|
+
} catch {
|
|
100
|
+
// kitty-graphics-agent not installed, skip
|
|
101
|
+
console.log('kitty-graphics-agent not installed, skipping round-trip test')
|
|
102
|
+
return
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
const sequence = buildKittySequence({ base64: TINY_PNG_B64 })
|
|
106
|
+
const output = `some text before\n${sequence}\nsome text after`
|
|
107
|
+
const result = extractKittyGraphics(output)
|
|
108
|
+
|
|
109
|
+
expect(result.cleanedOutput).toMatchInlineSnapshot(`
|
|
110
|
+
"some text before
|
|
111
|
+
|
|
112
|
+
some text after"
|
|
113
|
+
`)
|
|
114
|
+
expect(result.images.length).toBe(1)
|
|
115
|
+
expect(result.images[0].mime).toBe('image/png')
|
|
116
|
+
expect(result.images[0].data).toBe(TINY_PNG_B64)
|
|
117
|
+
})
|
|
118
|
+
|
|
119
|
+
test('chunked sequence round-trips correctly', async () => {
|
|
120
|
+
let extractKittyGraphics: typeof import('kitty-graphics-agent/parser').extractKittyGraphics
|
|
121
|
+
try {
|
|
122
|
+
const mod = await import('kitty-graphics-agent/parser')
|
|
123
|
+
extractKittyGraphics = mod.extractKittyGraphics
|
|
124
|
+
} catch {
|
|
125
|
+
console.log('kitty-graphics-agent not installed, skipping round-trip test')
|
|
126
|
+
return
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const bigPayload = 'A'.repeat(10000)
|
|
130
|
+
const sequence = buildKittySequence({ base64: bigPayload })
|
|
131
|
+
const result = extractKittyGraphics(sequence)
|
|
132
|
+
|
|
133
|
+
expect(result.cleanedOutput).toBe('')
|
|
134
|
+
expect(result.images.length).toBe(1)
|
|
135
|
+
expect(result.images[0].mime).toBe('image/png')
|
|
136
|
+
expect(result.images[0].data).toBe(bigPayload)
|
|
137
|
+
})
|
|
138
|
+
})
|
|
139
|
+
})
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
// Kitty Graphics Protocol emitter for CLI output.
|
|
2
|
+
// Emits base64-encoded PNG images as APC escape sequences so agents with
|
|
3
|
+
// kitty-graphics-agent (or compatible parsers) can extract and pass them
|
|
4
|
+
// to LLMs as media parts.
|
|
5
|
+
//
|
|
6
|
+
// Protocol format:
|
|
7
|
+
// \x1b_G<control_data>;<base64_payload>\x1b\\
|
|
8
|
+
//
|
|
9
|
+
// Only emits when AGENT_GRAPHICS env var contains 'kitty', signaling that
|
|
10
|
+
// an agent is intercepting stdout and can handle Kitty Graphics Protocol.
|
|
11
|
+
// See: https://github.com/remorses/kitty-graphics-agent
|
|
12
|
+
|
|
13
|
+
// Kitty spec recommends max 4096 bytes per chunk payload
|
|
14
|
+
const CHUNK_SIZE = 4096
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Check if the current environment supports Kitty Graphics Protocol output.
|
|
18
|
+
* Returns true when AGENT_GRAPHICS=kitty is set (agent is intercepting stdout).
|
|
19
|
+
*/
|
|
20
|
+
export function canEmitKittyGraphics(): boolean {
|
|
21
|
+
return process.env.AGENT_GRAPHICS?.includes('kitty') ?? false
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Emit a PNG image to stdout using the Kitty Graphics Protocol.
|
|
26
|
+
* The image is chunked per the spec (4096 bytes per chunk).
|
|
27
|
+
*
|
|
28
|
+
* Only call this when canEmitKittyGraphics() returns true.
|
|
29
|
+
*/
|
|
30
|
+
export function emitKittyImage({ base64 }: { base64: string }): void {
|
|
31
|
+
const chunks = splitIntoChunks(base64, CHUNK_SIZE)
|
|
32
|
+
|
|
33
|
+
if (chunks.length === 1) {
|
|
34
|
+
// Single chunk: no chunked transfer needed
|
|
35
|
+
process.stdout.write(`\x1b_Ga=T,f=100;${chunks[0]}\x1b\\`)
|
|
36
|
+
return
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// Multi-chunk: first chunk has full control data + m=1,
|
|
40
|
+
// continuation chunks have only m=1, last chunk has m=0
|
|
41
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
42
|
+
const isFirst = i === 0
|
|
43
|
+
const isLast = i === chunks.length - 1
|
|
44
|
+
const control = isFirst ? 'a=T,f=100,m=1' : isLast ? 'm=0' : 'm=1'
|
|
45
|
+
process.stdout.write(`\x1b_G${control};${chunks[i]}\x1b\\`)
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function splitIntoChunks(str: string, size: number): string[] {
|
|
50
|
+
if (str.length <= size) {
|
|
51
|
+
return [str]
|
|
52
|
+
}
|
|
53
|
+
const chunks: string[] = []
|
|
54
|
+
for (let i = 0; i < str.length; i += size) {
|
|
55
|
+
chunks.push(str.slice(i, i + size))
|
|
56
|
+
}
|
|
57
|
+
return chunks
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Build the raw Kitty Graphics escape sequence string for a PNG image.
|
|
62
|
+
* Useful for testing without writing to stdout.
|
|
63
|
+
*/
|
|
64
|
+
export function buildKittySequence({ base64 }: { base64: string }): string {
|
|
65
|
+
const chunks = splitIntoChunks(base64, CHUNK_SIZE)
|
|
66
|
+
|
|
67
|
+
if (chunks.length === 1) {
|
|
68
|
+
return `\x1b_Ga=T,f=100;${chunks[0]}\x1b\\`
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
const parts: string[] = []
|
|
72
|
+
for (let i = 0; i < chunks.length; i++) {
|
|
73
|
+
const isFirst = i === 0
|
|
74
|
+
const isLast = i === chunks.length - 1
|
|
75
|
+
const control = isFirst ? 'a=T,f=100,m=1' : isLast ? 'm=0' : 'm=1'
|
|
76
|
+
parts.push(`\x1b_G${control};${chunks[i]}\x1b\\`)
|
|
77
|
+
}
|
|
78
|
+
return parts.join('')
|
|
79
|
+
}
|