pi-browser-use 0.2.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -1
- package/dist/artifacts.d.ts +10 -0
- package/dist/artifacts.d.ts.map +1 -0
- package/dist/artifacts.js +21 -0
- package/dist/artifacts.js.map +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +85 -2
- package/dist/index.js.map +1 -1
- package/dist/tool-augment.d.ts +2 -0
- package/dist/tool-augment.d.ts.map +1 -1
- package/dist/tool-augment.js +15 -0
- package/dist/tool-augment.js.map +1 -1
- package/package.json +1 -1
- package/skills/playwright-handoff/SKILL.md +74 -0
- package/skills/triage-console/SKILL.md +47 -0
- package/skills/visual-qa/SKILL.md +37 -0
package/README.md
CHANGED
|
@@ -44,7 +44,18 @@ Hosts where `process.execPath` is not a directly executable Node runtime can set
|
|
|
44
44
|
|
|
45
45
|
## Page-scoped tools
|
|
46
46
|
|
|
47
|
-
Call `browser_list_pages` first, then pass its numeric `pageId` to page-scoped tools. See `skills/browser-policy/SKILL.md` for the full agent policy.
|
|
47
|
+
Call `browser_list_pages` first, then pass its numeric `pageId` to page-scoped tools. Click-family calls blocked by an overlay are retried once automatically after dismissing it with Escape. See `skills/browser-policy/SKILL.md` for the full agent policy.
|
|
48
|
+
|
|
49
|
+
## Saved artifacts
|
|
50
|
+
|
|
51
|
+
`browser_save_artifact` writes a screenshot or the rendered HTML to disk (default `~/.pi/browser-artifacts/`) and returns the path. Prefer it over inline captures for evidence, visual QA, and artifact sharing.
|
|
52
|
+
|
|
53
|
+
## Bundled skills
|
|
54
|
+
|
|
55
|
+
- `browser-policy` — CLI-first decision order, session modes, bot-wall and safety rules.
|
|
56
|
+
- `playwright-handoff` — when to stop clicking and run a repo Playwright spec instead (npm-first, bun alternatives, no repo changes required).
|
|
57
|
+
- `triage-console` — snapshot → console errors → failed network → screenshot.
|
|
58
|
+
- `visual-qa` — viewport matrix and canvas/WebGL discipline.
|
|
48
59
|
|
|
49
60
|
## Vision model (optional)
|
|
50
61
|
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export type ArtifactKind = 'screenshot' | 'html';
|
|
2
|
+
export declare function defaultArtifactDir(): string;
|
|
3
|
+
/** Resolve the destination file: explicit path wins, otherwise a timestamped file in the default dir. */
|
|
4
|
+
export declare function resolveArtifactTarget(kind: ArtifactKind, path?: unknown): string;
|
|
5
|
+
/** Pick the first image payload out of MCP content, if any. */
|
|
6
|
+
export declare function pickImageData(content: unknown): {
|
|
7
|
+
data: string;
|
|
8
|
+
mimeType: string;
|
|
9
|
+
} | undefined;
|
|
10
|
+
//# sourceMappingURL=artifacts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"artifacts.d.ts","sourceRoot":"","sources":["../src/artifacts.ts"],"names":[],"mappings":"AAGA,MAAM,MAAM,YAAY,GAAG,YAAY,GAAG,MAAM,CAAA;AAEhD,wBAAgB,kBAAkB,IAAI,MAAM,CAE3C;AAED,yGAAyG;AACzG,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,YAAY,EAAE,IAAI,CAAC,EAAE,OAAO,GAAG,MAAM,CAGhF;AASD,+DAA+D;AAC/D,wBAAgB,aAAa,CAAC,OAAO,EAAE,OAAO,GAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,QAAQ,EAAE,MAAM,CAAA;CAAE,GAAG,SAAS,CAK9F"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { homedir } from 'node:os';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
export function defaultArtifactDir() {
|
|
4
|
+
return join(homedir(), '.pi', 'browser-artifacts');
|
|
5
|
+
}
|
|
6
|
+
/** Resolve the destination file: explicit path wins, otherwise a timestamped file in the default dir. */
|
|
7
|
+
export function resolveArtifactTarget(kind, path) {
|
|
8
|
+
if (typeof path === 'string' && path.length > 0)
|
|
9
|
+
return path;
|
|
10
|
+
return join(defaultArtifactDir(), `page-${Date.now()}.${kind === 'html' ? 'html' : 'png'}`);
|
|
11
|
+
}
|
|
12
|
+
/** Pick the first image payload out of MCP content, if any. */
|
|
13
|
+
export function pickImageData(content) {
|
|
14
|
+
if (!Array.isArray(content))
|
|
15
|
+
return undefined;
|
|
16
|
+
const found = content.find((item) => item.type === 'image' && item.data);
|
|
17
|
+
if (!found?.data)
|
|
18
|
+
return undefined;
|
|
19
|
+
return { data: found.data, mimeType: found.mimeType ?? 'image/png' };
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=artifacts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"artifacts.js","sourceRoot":"","sources":["../src/artifacts.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAIhC,MAAM,UAAU,kBAAkB;IAChC,OAAO,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,mBAAmB,CAAC,CAAA;AACpD,CAAC;AAED,yGAAyG;AACzG,MAAM,UAAU,qBAAqB,CAAC,IAAkB,EAAE,IAAc;IACtE,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC5D,OAAO,IAAI,CAAC,kBAAkB,EAAE,EAAE,QAAQ,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAA;AAC7F,CAAC;AASD,+DAA+D;AAC/D,MAAM,UAAU,aAAa,CAAC,OAAgB;IAC5C,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,SAAS,CAAA;IAC7C,MAAM,KAAK,GAAI,OAAyB,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,IAAI,CAAC,CAAA;IAC3F,IAAI,CAAC,KAAK,EAAE,IAAI;QAAE,OAAO,SAAS,CAAA;IAClC,OAAO,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,IAAI,WAAW,EAAE,CAAA;AACtE,CAAC"}
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAsBA,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAkCzD,UAAU,aAAa;IACrB,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAA;IAChD,mBAAmB,CACjB,KAAK,EAAE,OAAO,GACb,OAAO,CACR;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAC/F,CAAA;CACF;AAED,UAAU,aAAa;IACrB,aAAa,CAAC,EAAE,aAAa,CAAA;CAC9B;AAED,UAAU,EAAE;IACV,YAAY,CAAC,GAAG,EAAE;QAChB,IAAI,EAAE,MAAM,CAAA;QACZ,KAAK,EAAE,MAAM,CAAA;QACb,WAAW,EAAE,MAAM,CAAA;QACnB,UAAU,EAAE,OAAO,CAAA;QACnB,OAAO,EAAE,CACP,UAAU,EAAE,MAAM,EAClB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,MAAM,CAAC,EAAE,WAAW,EACpB,QAAQ,CAAC,EAAE,OAAO,EAClB,GAAG,CAAC,EAAE,aAAa,KAChB,OAAO,CAAC;YACX,OAAO,EAAE,KAAK,CAAC;gBAAE,IAAI,EAAE,MAAM,CAAC;gBAAC,IAAI,CAAC,EAAE,MAAM,CAAC;gBAAC,IAAI,CAAC,EAAE,MAAM,CAAC;gBAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;aAAE,CAAC,CAAA;YACjF,OAAO,CAAC,EAAE,OAAO,CAAA;YACjB,OAAO,CAAC,EAAE,SAAS,CAAA;SACpB,CAAC,CAAA;KACH,GAAG,IAAI,CAAA;IACR,EAAE,CACA,KAAK,EAAE,MAAM,EACb,OAAO,EAAE,CAAC,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE;QAAE,GAAG,EAAE,MAAM,CAAA;KAAE,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,GACzF,IAAI,CAAA;CACR;AA8BD;;;;;;;;GAQG;AACH,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,EAAE,EAAE,EAAE,QA2KjD"}
|
package/dist/index.js
CHANGED
|
@@ -1,13 +1,20 @@
|
|
|
1
|
+
import { mkdirSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { dirname, join } from 'node:path';
|
|
1
4
|
import { Type } from 'typebox';
|
|
2
5
|
import { DevToolsClient } from './client.js';
|
|
3
6
|
import { resolveConfig } from './config.js';
|
|
4
7
|
import { isProjectTrusted, loadConfig } from './settings.js';
|
|
5
|
-
import { augmentToolDescription, extractTextContent, postProcessToolResult, } from './tool-augment.js';
|
|
8
|
+
import { augmentToolDescription, extractTextContent, looksOverlayBlocked, OVERLAY_RECOVERABLE, postProcessToolResult, } from './tool-augment.js';
|
|
9
|
+
import { pickImageData, resolveArtifactTarget } from './artifacts.js';
|
|
6
10
|
import { prepareBrowserProfile } from './profile.js';
|
|
7
11
|
import { createRegistryVisionCaller, handleAnalyzeScreenshot, } from './vision.js';
|
|
8
12
|
export { configToArgs, resolveConfig } from './config.js';
|
|
9
13
|
// All upstream tools are re-exported with this prefix to avoid name collisions.
|
|
10
14
|
const TOOL_PREFIX = 'browser_';
|
|
15
|
+
async function callUpstream(client, name, params, signal) {
|
|
16
|
+
return (await client.callTool(name, params, signal));
|
|
17
|
+
}
|
|
11
18
|
// Noisy, slow, or privileged upstream tools; skipped during registration.
|
|
12
19
|
const EXCLUDED_TOOLS = new Set([
|
|
13
20
|
'lighthouse_audit',
|
|
@@ -79,12 +86,87 @@ export default function browserUseExtension(pi) {
|
|
|
79
86
|
parameters: Type.Unsafe(tool.inputSchema),
|
|
80
87
|
async execute(_toolCallId, params, signal) {
|
|
81
88
|
await ensureConnected(signal);
|
|
82
|
-
const
|
|
89
|
+
const browser = client;
|
|
90
|
+
let result = await callUpstream(browser, originalName, params, signal);
|
|
91
|
+
if (result.isError &&
|
|
92
|
+
OVERLAY_RECOVERABLE.has(originalName) &&
|
|
93
|
+
looksOverlayBlocked(extractTextContent(result.content))) {
|
|
94
|
+
// One recovery attempt: dismiss the overlay, then retry the
|
|
95
|
+
// original call. Any failure here falls through to the
|
|
96
|
+
// original error, which already carries a hint.
|
|
97
|
+
try {
|
|
98
|
+
const escapeArgs = typeof params.pageId === 'number'
|
|
99
|
+
? { pageId: params.pageId, key: 'Escape' }
|
|
100
|
+
: { key: 'Escape' };
|
|
101
|
+
await callUpstream(browser, 'press_key', escapeArgs, signal);
|
|
102
|
+
result = await callUpstream(browser, originalName, params, signal);
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
// Fall through to the original result below.
|
|
106
|
+
}
|
|
107
|
+
}
|
|
83
108
|
return { ...toToolContent(result, originalName), details: undefined };
|
|
84
109
|
},
|
|
85
110
|
});
|
|
86
111
|
}
|
|
87
112
|
}
|
|
113
|
+
function registerSaveArtifactTool() {
|
|
114
|
+
const properties = {
|
|
115
|
+
kind: Type.Union([Type.Literal('screenshot'), Type.Literal('html')], {
|
|
116
|
+
description: 'Capture a viewport screenshot (PNG) or the full rendered HTML.',
|
|
117
|
+
}),
|
|
118
|
+
path: Type.Optional(Type.String({
|
|
119
|
+
description: 'Absolute destination path. Defaults to ~/.pi/browser-artifacts/<timestamp>.<png|html>.',
|
|
120
|
+
})),
|
|
121
|
+
};
|
|
122
|
+
if (config?.experimentalPageIdRouting === true) {
|
|
123
|
+
properties.pageId = Type.Number({
|
|
124
|
+
description: 'Numeric page ID returned by browser_list_pages.',
|
|
125
|
+
});
|
|
126
|
+
}
|
|
127
|
+
pi.registerTool({
|
|
128
|
+
name: `${TOOL_PREFIX}save_artifact`,
|
|
129
|
+
label: `${TOOL_PREFIX}save_artifact`,
|
|
130
|
+
description: 'Save a screenshot or the rendered HTML of the current page to disk and return its path. Prefer this over pulling image bytes into context when the capture is evidence (bug reports, visual QA, artifact sharing) rather than something you need to look at right now.',
|
|
131
|
+
parameters: Type.Object(properties),
|
|
132
|
+
async execute(_toolCallId, params, signal) {
|
|
133
|
+
await ensureConnected(signal);
|
|
134
|
+
const browser = client;
|
|
135
|
+
const kind = params.kind === 'html' ? 'html' : 'screenshot';
|
|
136
|
+
const pageId = typeof params.pageId === 'number' ? params.pageId : undefined;
|
|
137
|
+
const target = resolveArtifactTarget(kind, params.path);
|
|
138
|
+
if (kind === 'html') {
|
|
139
|
+
const pageArgs = pageId === undefined ? {} : { pageId };
|
|
140
|
+
const evaluated = (await browser.callTool('evaluate_script', {
|
|
141
|
+
...pageArgs,
|
|
142
|
+
function: '() => document.documentElement.outerHTML',
|
|
143
|
+
}, signal));
|
|
144
|
+
const html = extractTextContent(evaluated.content);
|
|
145
|
+
if (!html)
|
|
146
|
+
throw new Error('Page HTML came back empty.');
|
|
147
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
148
|
+
writeFileSync(target, html, 'utf8');
|
|
149
|
+
return {
|
|
150
|
+
content: [
|
|
151
|
+
{ type: 'text', text: `Saved page HTML (${html.length} chars) to ${target}` },
|
|
152
|
+
],
|
|
153
|
+
details: undefined,
|
|
154
|
+
};
|
|
155
|
+
}
|
|
156
|
+
const shotArgs = pageId === undefined ? {} : { pageId };
|
|
157
|
+
const shot = (await browser.callTool('take_screenshot', shotArgs, signal));
|
|
158
|
+
const image = pickImageData(shot.content);
|
|
159
|
+
if (!image)
|
|
160
|
+
throw new Error('Screenshot came back without image data.');
|
|
161
|
+
mkdirSync(dirname(target), { recursive: true });
|
|
162
|
+
writeFileSync(target, Buffer.from(image.data, 'base64'));
|
|
163
|
+
return {
|
|
164
|
+
content: [{ type: 'text', text: `Saved screenshot to ${target}` }],
|
|
165
|
+
details: undefined,
|
|
166
|
+
};
|
|
167
|
+
},
|
|
168
|
+
});
|
|
169
|
+
}
|
|
88
170
|
async function registerVisionTool(visionConfig) {
|
|
89
171
|
const properties = {
|
|
90
172
|
instruction: Type.Optional(Type.String({
|
|
@@ -119,6 +201,7 @@ export default function browserUseExtension(pi) {
|
|
|
119
201
|
prepareBrowserProfile(config);
|
|
120
202
|
client = new DevToolsClient(config);
|
|
121
203
|
await registerUpstreamTools();
|
|
204
|
+
registerSaveArtifactTool();
|
|
122
205
|
if (config.visionModel) {
|
|
123
206
|
await registerVisionTool(config.visionModel);
|
|
124
207
|
}
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAgB,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAyB,MAAM,aAAa,CAAA;AAClE,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC5D,OAAO,EACL,sBAAsB,EACtB,kBAAkB,EAClB,qBAAqB,GACtB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AACpD,OAAO,EACL,0BAA0B,EAC1B,uBAAuB,GAExB,MAAM,aAAa,CAAA;AAEpB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAEzD,gFAAgF;AAChF,MAAM,WAAW,GAAG,UAAU,CAAA;
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAA;AAClD,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AACjC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AACzC,OAAO,EAAE,IAAI,EAAgB,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAyB,MAAM,aAAa,CAAA;AAClE,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC5D,OAAO,EACL,sBAAsB,EACtB,kBAAkB,EAClB,mBAAmB,EACnB,mBAAmB,EACnB,qBAAqB,GACtB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAqB,MAAM,gBAAgB,CAAA;AACxF,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AACpD,OAAO,EACL,0BAA0B,EAC1B,uBAAuB,GAExB,MAAM,aAAa,CAAA;AAEpB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAEzD,gFAAgF;AAChF,MAAM,WAAW,GAAG,UAAU,CAAA;AAO9B,KAAK,UAAU,YAAY,CACzB,MAAsB,EACtB,IAAY,EACZ,MAA+B,EAC/B,MAAoB;IAEpB,OAAO,CAAC,MAAM,MAAM,CAAC,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,CAAC,CAAmB,CAAA;AACxE,CAAC;AAED,0EAA0E;AAC1E,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,kBAAkB;IAClB,6BAA6B;IAC7B,yBAAyB;IACzB,wBAAwB;IACxB,kBAAkB;IAClB,iBAAiB;IACjB,mBAAmB;IACnB,iBAAiB;IACjB,kBAAkB;IAClB,0BAA0B;IAC1B,qBAAqB;CACtB,CAAC,CAAA;AAuCF,SAAS,aAAa,CACpB,MAGC,EACD,YAAoB;IAEpB,MAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACtD,MAAM,SAAS,GAAG,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;IAClE,MAAM,OAAO,GAA6E,EAAE,CAAA;IAC5F,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAA;IACjD,CAAC;SAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;QACxF,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACvC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,WAAW,EAAE,CAAC,CAAA;YAC1F,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAA;IAClE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAa,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAA;AAC3E,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,EAAM;IAChD,IAAI,MAAoC,CAAA;IACxC,IAAI,MAAkC,CAAA;IAEtC,KAAK,UAAU,eAAe,CAAC,MAAoB;QACjD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;QAChE,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,UAAU,qBAAqB;QAClC,MAAM,eAAe,EAAE,CAAA;QACvB,MAAM,aAAa,GAAG,MAAM,MAAO,CAAC,YAAY,EAAE,CAAA;QAClD,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAQ;YAC3C,MAAM,YAAY,GAAG,GAAG,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YACjD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAA;YAC9B,MAAM,WAAW,GAAG,sBAAsB,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;YAChF,EAAE,CAAC,YAAY,CAAC;gBACd,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,YAAY;gBACnB,WAAW;gBACX,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAsB,CAAC;gBACpD,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM;oBACvC,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;oBAC7B,MAAM,OAAO,GAAG,MAAO,CAAA;oBACvB,IAAI,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;oBACtE,IACE,MAAM,CAAC,OAAO;wBACd,mBAAmB,CAAC,GAAG,CAAC,YAAY,CAAC;wBACrC,mBAAmB,CAAC,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,EACvD,CAAC;wBACD,4DAA4D;wBAC5D,uDAAuD;wBACvD,gDAAgD;wBAChD,IAAI,CAAC;4BACH,MAAM,UAAU,GACd,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ;gCAC/B,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,EAAE,QAAQ,EAAE;gCAC1C,CAAC,CAAC,EAAE,GAAG,EAAE,QAAQ,EAAE,CAAA;4BACvB,MAAM,YAAY,CAAC,OAAO,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,CAAC,CAAA;4BAC5D,MAAM,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,CAAA;wBACpE,CAAC;wBAAC,MAAM,CAAC;4BACP,6CAA6C;wBAC/C,CAAC;oBACH,CAAC;oBACD,OAAO,EAAE,GAAG,aAAa,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAA;gBACvE,CAAC;aACF,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,SAAS,wBAAwB;QAC/B,MAAM,UAAU,GAA4B;YAC1C,IAAI,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,EAAE;gBACnE,WAAW,EAAE,gEAAgE;aAC9E,CAAC;YACF,IAAI,EAAE,IAAI,CAAC,QAAQ,CACjB,IAAI,CAAC,MAAM,CAAC;gBACV,WAAW,EACT,wFAAwF;aAC3F,CAAC,CACH;SACF,CAAA;QACD,IAAI,MAAM,EAAE,yBAAyB,KAAK,IAAI,EAAE,CAAC;YAC/C,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;gBAC9B,WAAW,EAAE,iDAAiD;aAC/D,CAAC,CAAA;QACJ,CAAC;QACD,EAAE,CAAC,YAAY,CAAC;YACd,IAAI,EAAE,GAAG,WAAW,eAAe;YACnC,KAAK,EAAE,GAAG,WAAW,eAAe;YACpC,WAAW,EACT,wQAAwQ;YAC1Q,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;YACnC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM;gBACvC,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;gBAC7B,MAAM,OAAO,GAAG,MAAO,CAAA;gBACvB,MAAM,IAAI,GAAiB,MAAM,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY,CAAA;gBACzE,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAA;gBAC5E,MAAM,MAAM,GAAG,qBAAqB,CAAC,IAAI,EAAE,MAAM,CAAC,IAAI,CAAC,CAAA;gBACvD,IAAI,IAAI,KAAK,MAAM,EAAE,CAAC;oBACpB,MAAM,QAAQ,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAA;oBACvD,MAAM,SAAS,GAAG,CAAC,MAAM,OAAO,CAAC,QAAQ,CACvC,iBAAiB,EACjB;wBACE,GAAG,QAAQ;wBACX,QAAQ,EAAE,0CAA0C;qBACrD,EACD,MAAM,CACP,CAAmB,CAAA;oBACpB,MAAM,IAAI,GAAG,kBAAkB,CAAC,SAAS,CAAC,OAAO,CAAC,CAAA;oBAClD,IAAI,CAAC,IAAI;wBAAE,MAAM,IAAI,KAAK,CAAC,4BAA4B,CAAC,CAAA;oBACxD,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;oBAC/C,aAAa,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM,CAAC,CAAA;oBACnC,OAAO;wBACL,OAAO,EAAE;4BACP,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,oBAAoB,IAAI,CAAC,MAAM,cAAc,MAAM,EAAE,EAAE;yBAC9E;wBACD,OAAO,EAAE,SAAS;qBACnB,CAAA;gBACH,CAAC;gBACD,MAAM,QAAQ,GAAG,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAA;gBACvD,MAAM,IAAI,GAAG,CAAC,MAAM,OAAO,CAAC,QAAQ,CAAC,iBAAiB,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAmB,CAAA;gBAC5F,MAAM,KAAK,GAAG,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;gBACzC,IAAI,CAAC,KAAK;oBAAE,MAAM,IAAI,KAAK,CAAC,0CAA0C,CAAC,CAAA;gBACvE,SAAS,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;gBAC/C,aAAa,CAAC,MAAM,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAAA;gBACxD,OAAO;oBACL,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,uBAAuB,MAAM,EAAE,EAAE,CAAC;oBAClE,OAAO,EAAE,SAAS;iBACnB,CAAA;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,UAAU,kBAAkB,CAAC,YAA+B;QAC/D,MAAM,UAAU,GAA4B;YAC1C,WAAW,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,MAAM,CAAC;gBACV,WAAW,EACT,+FAA+F;aAClG,CAAC,CACH;SACF,CAAA;QACD,IAAI,MAAM,EAAE,yBAAyB,KAAK,IAAI,EAAE,CAAC;YAC/C,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;gBAC9B,WAAW,EAAE,iDAAiD;aAC/D,CAAC,CAAA;QACJ,CAAC;QACD,EAAE,CAAC,YAAY,CAAC;YACd,IAAI,EAAE,GAAG,WAAW,oBAAoB;YACxC,KAAK,EAAE,GAAG,WAAW,oBAAoB;YACzC,WAAW,EACT,6PAA6P;YAC/P,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;YACnC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG;gBACvD,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;gBAC7B,IAAI,CAAC,GAAG,EAAE,aAAa;oBACrB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;gBAC1E,MAAM,UAAU,GAAG,0BAA0B,CAAC,YAAY,EAAE,GAAG,CAAC,aAAa,CAAC,CAAA;gBAC9E,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAA;gBAC5E,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAC1C;oBACE,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,MAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC;iBACjE,EACD,UAAU,EACV,EAAE,WAAW,EAAE,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,EACzF,MAAM,CACP,CAAA;gBACD,OAAO,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAA;YAC1C,CAAC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,EAAE,CAAC,EAAE,CAAC,eAAe,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;QAC3C,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,cAAc,EAAE,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAC3F,qBAAqB,CAAC,MAAM,CAAC,CAAA;QAC7B,MAAM,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,CAAA;QACnC,MAAM,qBAAqB,EAAE,CAAA;QAC7B,wBAAwB,EAAE,CAAA;QAC1B,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,MAAM,kBAAkB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QAC9C,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QACnC,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACpB,MAAM,GAAG,SAAS,CAAA;QACpB,CAAC;IACH,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
package/dist/tool-augment.d.ts
CHANGED
|
@@ -1,4 +1,6 @@
|
|
|
1
1
|
export declare function augmentToolDescription(prefixedName: string, description: string): string;
|
|
2
2
|
export declare function postProcessToolResult(originalName: string, text: string): string;
|
|
3
|
+
export declare const OVERLAY_RECOVERABLE: Set<string>;
|
|
4
|
+
export declare function looksOverlayBlocked(text: string): boolean;
|
|
3
5
|
export declare function extractTextContent(content: unknown): string;
|
|
4
6
|
//# sourceMappingURL=tool-augment.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-augment.d.ts","sourceRoot":"","sources":["../src/tool-augment.ts"],"names":[],"mappings":"AASA,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAGxF;AAED,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAQhF;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAS3D"}
|
|
1
|
+
{"version":3,"file":"tool-augment.d.ts","sourceRoot":"","sources":["../src/tool-augment.ts"],"names":[],"mappings":"AASA,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAGxF;AAED,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAQhF;AAID,eAAO,MAAM,mBAAmB,aAS9B,CAAA;AAEF,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAEzD;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAS3D"}
|
package/dist/tool-augment.js
CHANGED
|
@@ -18,6 +18,21 @@ export function postProcessToolResult(originalName, text) {
|
|
|
18
18
|
}
|
|
19
19
|
return text;
|
|
20
20
|
}
|
|
21
|
+
// Click-family tools worth one automatic recovery attempt when an overlay
|
|
22
|
+
// blocks them: dismiss with Escape, then retry once.
|
|
23
|
+
export const OVERLAY_RECOVERABLE = new Set([
|
|
24
|
+
'click',
|
|
25
|
+
'click_at',
|
|
26
|
+
'fill',
|
|
27
|
+
'fill_form',
|
|
28
|
+
'press_key',
|
|
29
|
+
'type_text',
|
|
30
|
+
'hover',
|
|
31
|
+
'drag',
|
|
32
|
+
]);
|
|
33
|
+
export function looksOverlayBlocked(text) {
|
|
34
|
+
return /overlay|obscured|intercept|behind another element|not clickable|not visible/i.test(text);
|
|
35
|
+
}
|
|
21
36
|
export function extractTextContent(content) {
|
|
22
37
|
if (!Array.isArray(content))
|
|
23
38
|
return '';
|
package/dist/tool-augment.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"tool-augment.js","sourceRoot":"","sources":["../src/tool-augment.ts"],"names":[],"mappings":"AAAA,MAAM,KAAK,GAA2B;IACpC,aAAa,EACX,mGAAmG;IACrG,YAAY,EAAE,+EAA+E;IAC7F,iBAAiB,EAAE,2DAA2D;IAC9E,qBAAqB,EAAE,gEAAgE;IACvF,qBAAqB,EAAE,0DAA0D;CAClF,CAAA;AAED,MAAM,UAAU,sBAAsB,CAAC,YAAoB,EAAE,WAAmB;IAC9E,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,CAAA;IAChC,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,WAAW,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,WAAW,CAAA;AAC/D,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,YAAoB,EAAE,IAAY;IACtE,IAAI,YAAY,KAAK,OAAO,IAAI,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACzE,OAAO,GAAG,IAAI,oGAAoG,CAAA;IACpH,CAAC;IACD,IAAI,qCAAqC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAClF,OAAO,GAAG,IAAI,kFAAkF,CAAA;IAClG,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAgB;IACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAA;IACtC,OAAO,OAAO;SACX,MAAM,CACL,CAAC,IAAI,EAA2C,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,CAC7F;SACA,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;SACvE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAc,CAAC;SAClC,IAAI,CAAC,IAAI,CAAC,CAAA;AACf,CAAC"}
|
|
1
|
+
{"version":3,"file":"tool-augment.js","sourceRoot":"","sources":["../src/tool-augment.ts"],"names":[],"mappings":"AAAA,MAAM,KAAK,GAA2B;IACpC,aAAa,EACX,mGAAmG;IACrG,YAAY,EAAE,+EAA+E;IAC7F,iBAAiB,EAAE,2DAA2D;IAC9E,qBAAqB,EAAE,gEAAgE;IACvF,qBAAqB,EAAE,0DAA0D;CAClF,CAAA;AAED,MAAM,UAAU,sBAAsB,CAAC,YAAoB,EAAE,WAAmB;IAC9E,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,CAAA;IAChC,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,WAAW,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,WAAW,CAAA;AAC/D,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,YAAoB,EAAE,IAAY;IACtE,IAAI,YAAY,KAAK,OAAO,IAAI,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACzE,OAAO,GAAG,IAAI,oGAAoG,CAAA;IACpH,CAAC;IACD,IAAI,qCAAqC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAClF,OAAO,GAAG,IAAI,kFAAkF,CAAA;IAClG,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,0EAA0E;AAC1E,qDAAqD;AACrD,MAAM,CAAC,MAAM,mBAAmB,GAAG,IAAI,GAAG,CAAC;IACzC,OAAO;IACP,UAAU;IACV,MAAM;IACN,WAAW;IACX,WAAW;IACX,WAAW;IACX,OAAO;IACP,MAAM;CACP,CAAC,CAAA;AAEF,MAAM,UAAU,mBAAmB,CAAC,IAAY;IAC9C,OAAO,8EAA8E,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAClG,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAgB;IACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAA;IACtC,OAAO,OAAO;SACX,MAAM,CACL,CAAC,IAAI,EAA2C,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,CAC7F;SACA,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;SACvE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAc,CAAC;SAClC,IAAI,CAAC,IAAI,CAAC,CAAA;AACf,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-browser-use",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Opinionated browser-use for the Pi coding agent, powered by chrome-devtools-mcp (not Playwright). Fresh headless by default, authenticated persistent profile opt-in, CLI-first policy bundled.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package",
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: playwright-handoff
|
|
3
|
+
description: Decide when browser automation has hit its limits and hand off to a repo Playwright run instead. Use when a task needs request mocking, assertions, traces, repeatability, or CI parity that browser_* tools cannot provide.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Playwright Handoff
|
|
7
|
+
|
|
8
|
+
`browser_*` drives a real page well. It cannot do the following — hand these to Playwright instead of fighting the browser:
|
|
9
|
+
|
|
10
|
+
- **Request mocking** (`page.route` → `route.fulfill` for API responses)
|
|
11
|
+
- **Assertions with auto-retry** (`expect(locator).toBeVisible({ timeout })`)
|
|
12
|
+
- **Traces, video, isolated contexts per test**
|
|
13
|
+
- **Repeatable gates** that must pass in CI
|
|
14
|
+
|
|
15
|
+
## Rule
|
|
16
|
+
|
|
17
|
+
One failed browser attempt at the above is enough signal. Do not retry twice via `browser_*`; switch runners.
|
|
18
|
+
|
|
19
|
+
## Running without touching the repo
|
|
20
|
+
|
|
21
|
+
No install and no repo changes are required. Detect the runner first, then run a throwaway spec from `/tmp` against the already-running server or live URL:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
command -v bun >/dev/null 2>&1 && echo HAS_BUN || echo NPM_ONLY
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
```bash
|
|
28
|
+
# Repo already has Playwright (e.g. agent-hq, cortana): use it directly.
|
|
29
|
+
bunx playwright test --config=playwright.config.ts --grep "<test name>"
|
|
30
|
+
```
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
# Any other repo: npm cache + /tmp spec, zero repo mutation.
|
|
34
|
+
npx -y @playwright/test test /tmp/smoke.spec.ts --reporter=list
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
# Same via bun when available.
|
|
39
|
+
bunx --package @playwright/test playwright test /tmp/smoke.spec.ts --reporter=list
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
First run on a machine downloads the browser once to the shared user cache (`~/.cache/ms-playwright`, ~170MB); every repo after that reuses it.
|
|
43
|
+
|
|
44
|
+
## Minimal throwaway spec shape
|
|
45
|
+
|
|
46
|
+
```ts
|
|
47
|
+
// /tmp/smoke.spec.ts
|
|
48
|
+
import { expect, test } from '@playwright/test'
|
|
49
|
+
|
|
50
|
+
test('guest sees the workspace', async ({ page }) => {
|
|
51
|
+
await page.goto(process.env.SMOKE_URL ?? 'http://localhost:3000/')
|
|
52
|
+
await expect(page.getByRole('button', { name: 'User settings' })).toBeVisible({ timeout: 20000 })
|
|
53
|
+
})
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
SMOKE_URL=https://staging.example.com npx -y @playwright/test test /tmp/smoke.spec.ts
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
## Headless discipline
|
|
61
|
+
|
|
62
|
+
headed browsers pop windows and steal focus. Prefer headless for agent runs:
|
|
63
|
+
|
|
64
|
+
```bash
|
|
65
|
+
# npm / npx: headless is Playwright's default; keep it that way.
|
|
66
|
+
# bun repos whose config defaults to headed on macOS (e.g. agent-hq):
|
|
67
|
+
PLAYWRIGHT_HEADLESS=1 bunx playwright test --config=playwright.config.ts
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Keep headed runs for human-watched perf validation only.
|
|
71
|
+
|
|
72
|
+
## Reporting back
|
|
73
|
+
|
|
74
|
+
Summarize pass/fail plus the failing assertion or trace path. Delete nothing from the repo — the only artifacts are `/tmp/*.spec.ts` and the shared browser cache.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: triage-console
|
|
3
|
+
description: Standard flow for diagnosing a broken page. Use when a site misbehaves, renders wrong, or throws — snapshot, console errors, failed network requests, then screenshot.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Console Triage
|
|
7
|
+
|
|
8
|
+
Run in order. Stop at the first step that explains the breakage; do not skip ahead to screenshots.
|
|
9
|
+
|
|
10
|
+
## 1. Snapshot
|
|
11
|
+
|
|
12
|
+
```text
|
|
13
|
+
browser_list_pages({})
|
|
14
|
+
browser_take_snapshot({ "pageId": <id> })
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
If the expected element is missing from the tree, the page likely failed to load data or JS — continue. If it is present but misbehaving, skip to step 4.
|
|
18
|
+
|
|
19
|
+
## 2. Console errors
|
|
20
|
+
|
|
21
|
+
```text
|
|
22
|
+
browser_list_console_messages({ "pageId": <id> })
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
Read `error` entries first. A single `pageerror` often names the exact failing module. `warning` entries are context, not causes.
|
|
26
|
+
|
|
27
|
+
## 3. Failed network
|
|
28
|
+
|
|
29
|
+
```text
|
|
30
|
+
browser_list_network_requests({ "pageId": <id> })
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Look for non-2xx statuses, blocked URL patterns, CORS failures, and requests that never left `pending`. Correlate URLs with the console errors from step 2 before concluding.
|
|
34
|
+
|
|
35
|
+
## 4. Screenshot (evidence, not diagnosis)
|
|
36
|
+
|
|
37
|
+
Only now, and prefer saving over inlining:
|
|
38
|
+
|
|
39
|
+
```text
|
|
40
|
+
browser_save_artifact({ "pageId": <id>, "kind": "screenshot" })
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
Inline `browser_take_screenshot` only when you must look at the pixels yourself this turn.
|
|
44
|
+
|
|
45
|
+
## Reporting
|
|
46
|
+
|
|
47
|
+
One finding per cause, each with the exact console line or failing URL. If steps 1–3 are clean and the page still misbehaves, say so explicitly and hand off to `playwright-handoff` (mock the failing API, assert the render) instead of re-running triage.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: visual-qa
|
|
3
|
+
description: Screenshot discipline for visual checks of WebGL/canvas scenes and layouts. Use when asked whether something "looks right", for before/after comparisons, or for viewport coverage.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Visual QA
|
|
7
|
+
|
|
8
|
+
The accessibility tree cannot see canvas pixels. That is the only reason to look at screenshots — exhaust the tree first.
|
|
9
|
+
|
|
10
|
+
## Viewport matrix
|
|
11
|
+
|
|
12
|
+
Cover at least two sizes; small viewports catch most layout regressions:
|
|
13
|
+
|
|
14
|
+
```text
|
|
15
|
+
browser_resize_page({ "pageId": <id>, "width": 1280, "height": 720 })
|
|
16
|
+
browser_save_artifact({ "pageId": <id>, "kind": "screenshot" })
|
|
17
|
+
browser_resize_page({ "pageId": <id>, "width": 390, "height": 844 })
|
|
18
|
+
browser_save_artifact({ "pageId": <id>, "kind": "screenshot" })
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Save, don't inline — `browser_save_artifact` returns paths. Inline `browser_take_screenshot` only for the frame you are judging this turn.
|
|
22
|
+
|
|
23
|
+
## When the tree is blind (canvas/WebGL)
|
|
24
|
+
|
|
25
|
+
1. `browser_analyze_screenshot` (requires `visionModel` config) for coordinates or element identification the tree cannot provide.
|
|
26
|
+
2. Feed coordinates to `browser_click_at` — never guess pixels yourself.
|
|
27
|
+
3. If the scene needs deterministic frames (animations, thumbnails), stop: that is `playwright-handoff` territory (`waitForFunction`, fixed timeouts, traces).
|
|
28
|
+
|
|
29
|
+
## Failure bar
|
|
30
|
+
|
|
31
|
+
A visual failure needs: what was expected, what rendered instead, the artifact path, and the viewport. "Looks off" without an artifact path is not a finding.
|
|
32
|
+
|
|
33
|
+
## Pixel-flakiness notes
|
|
34
|
+
|
|
35
|
+
- WebGL anti-aliasing and font rendering differ between headless (SwiftShader) and GPU browsers — compare like with like.
|
|
36
|
+
- Allow one retry for animation timing before calling a failure.
|
|
37
|
+
- Animations mid-frame are not bugs; wait for idle, then capture.
|