pi-browser-use 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +69 -0
- package/dist/client.d.ts +29 -0
- package/dist/client.d.ts.map +1 -0
- package/dist/client.js +222 -0
- package/dist/client.js.map +1 -0
- package/dist/config.d.ts +46 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +120 -0
- package/dist/config.js.map +1 -0
- package/dist/index.d.ts +47 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +133 -0
- package/dist/index.js.map +1 -0
- package/dist/profile.d.ts +11 -0
- package/dist/profile.d.ts.map +1 -0
- package/dist/profile.js +58 -0
- package/dist/profile.js.map +1 -0
- package/dist/settings.d.ts +12 -0
- package/dist/settings.d.ts.map +1 -0
- package/dist/settings.js +52 -0
- package/dist/settings.js.map +1 -0
- package/dist/tool-augment.d.ts +4 -0
- package/dist/tool-augment.d.ts.map +1 -0
- package/dist/tool-augment.js +30 -0
- package/dist/tool-augment.js.map +1 -0
- package/dist/vision.d.ts +49 -0
- package/dist/vision.d.ts.map +1 -0
- package/dist/vision.js +144 -0
- package/dist/vision.js.map +1 -0
- package/package.json +76 -0
- package/skills/browser-policy/SKILL.md +33 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { Type } from 'typebox';
|
|
2
|
+
import { DevToolsClient } from './client.js';
|
|
3
|
+
import { resolveConfig } from './config.js';
|
|
4
|
+
import { isProjectTrusted, loadConfig } from './settings.js';
|
|
5
|
+
import { augmentToolDescription, extractTextContent, postProcessToolResult, } from './tool-augment.js';
|
|
6
|
+
import { prepareBrowserProfile } from './profile.js';
|
|
7
|
+
import { createRegistryVisionCaller, handleAnalyzeScreenshot, } from './vision.js';
|
|
8
|
+
export { configToArgs, resolveConfig } from './config.js';
|
|
9
|
+
// All upstream tools are re-exported with this prefix to avoid name collisions.
|
|
10
|
+
const TOOL_PREFIX = 'browser_';
|
|
11
|
+
// Noisy, slow, or privileged upstream tools; skipped during registration.
|
|
12
|
+
const EXCLUDED_TOOLS = new Set([
|
|
13
|
+
'lighthouse_audit',
|
|
14
|
+
'performance_analyze_insight',
|
|
15
|
+
'performance_start_trace',
|
|
16
|
+
'performance_stop_trace',
|
|
17
|
+
'screencast_start',
|
|
18
|
+
'screencast_stop',
|
|
19
|
+
'install_extension',
|
|
20
|
+
'list_extensions',
|
|
21
|
+
'reload_extension',
|
|
22
|
+
'trigger_extension_action',
|
|
23
|
+
'uninstall_extension',
|
|
24
|
+
]);
|
|
25
|
+
function toToolContent(result, originalName) {
|
|
26
|
+
const textContent = extractTextContent(result.content);
|
|
27
|
+
const processed = postProcessToolResult(originalName, textContent);
|
|
28
|
+
const content = [];
|
|
29
|
+
if (processed !== textContent) {
|
|
30
|
+
content.push({ type: 'text', text: processed });
|
|
31
|
+
}
|
|
32
|
+
else if (result.content) {
|
|
33
|
+
for (const item of result.content) {
|
|
34
|
+
if (item.type === 'text' && item.text)
|
|
35
|
+
content.push({ type: 'text', text: item.text });
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
if (result.content) {
|
|
39
|
+
for (const item of result.content) {
|
|
40
|
+
if (item.type === 'image' && item.data) {
|
|
41
|
+
content.push({ type: 'image', data: item.data, mimeType: item.mimeType ?? 'image/png' });
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
if (content.length === 0)
|
|
46
|
+
content.push({ type: 'text', text: '' });
|
|
47
|
+
return result.isError ? { content, isError: true } : { content };
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Pi extension entry point. On session_start spawns chrome-devtools-mcp,
|
|
51
|
+
* discovers upstream tools, and registers each as browser_*. On
|
|
52
|
+
* session_shutdown tears the subprocess down. Nothing runs persistently.
|
|
53
|
+
*
|
|
54
|
+
* Defaults are fresh headless (isolated ephemeral profile, no window). Set
|
|
55
|
+
* sessionMode "persistent" for the authenticated profile, or "existing" with
|
|
56
|
+
* autoConnect/browserUrl to drive an already-running Chrome.
|
|
57
|
+
*/
|
|
58
|
+
export default function browserUseExtension(pi) {
|
|
59
|
+
let config;
|
|
60
|
+
let client;
|
|
61
|
+
async function ensureConnected(signal) {
|
|
62
|
+
if (!client)
|
|
63
|
+
throw new Error('browser-use: session not started');
|
|
64
|
+
await client.ensureReady(signal);
|
|
65
|
+
}
|
|
66
|
+
async function registerUpstreamTools() {
|
|
67
|
+
await ensureConnected();
|
|
68
|
+
const upstreamTools = await client.listAllTools();
|
|
69
|
+
for (const tool of upstreamTools) {
|
|
70
|
+
if (EXCLUDED_TOOLS.has(tool.name))
|
|
71
|
+
continue;
|
|
72
|
+
const prefixedName = `${TOOL_PREFIX}${tool.name}`;
|
|
73
|
+
const originalName = tool.name;
|
|
74
|
+
const description = augmentToolDescription(prefixedName, tool.description ?? '');
|
|
75
|
+
pi.registerTool({
|
|
76
|
+
name: prefixedName,
|
|
77
|
+
label: prefixedName,
|
|
78
|
+
description,
|
|
79
|
+
parameters: Type.Unsafe(tool.inputSchema),
|
|
80
|
+
async execute(_toolCallId, params, signal) {
|
|
81
|
+
await ensureConnected(signal);
|
|
82
|
+
const result = (await client.callTool(originalName, params, signal));
|
|
83
|
+
return { ...toToolContent(result, originalName), details: undefined };
|
|
84
|
+
},
|
|
85
|
+
});
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
async function registerVisionTool(visionConfig) {
|
|
89
|
+
const properties = {
|
|
90
|
+
instruction: Type.Optional(Type.String({
|
|
91
|
+
description: 'What to identify or analyze visually (e.g. "Find the coordinates of the blue submit button").',
|
|
92
|
+
})),
|
|
93
|
+
};
|
|
94
|
+
if (config?.experimentalPageIdRouting === true) {
|
|
95
|
+
properties.pageId = Type.Number({
|
|
96
|
+
description: 'Numeric page ID returned by browser_list_pages.',
|
|
97
|
+
});
|
|
98
|
+
}
|
|
99
|
+
pi.registerTool({
|
|
100
|
+
name: `${TOOL_PREFIX}analyze_screenshot`,
|
|
101
|
+
label: `${TOOL_PREFIX}analyze_screenshot`,
|
|
102
|
+
description: 'Analyze the current page visually using a screenshot. Use when you need to identify elements by visual attributes (color, layout, position) not available in the accessibility tree, or when you need precise pixel coordinates for coordinate click tools.',
|
|
103
|
+
parameters: Type.Object(properties),
|
|
104
|
+
async execute(_toolCallId, params, signal, _onUpdate, ctx) {
|
|
105
|
+
await ensureConnected(signal);
|
|
106
|
+
if (!ctx?.modelRegistry)
|
|
107
|
+
throw new Error('Vision model registry is unavailable in this session.');
|
|
108
|
+
const callVision = createRegistryVisionCaller(visionConfig, ctx.modelRegistry);
|
|
109
|
+
const pageId = typeof params.pageId === 'number' ? params.pageId : undefined;
|
|
110
|
+
const result = await handleAnalyzeScreenshot({
|
|
111
|
+
callTool: (name, args, sig) => client.callTool(name, args, sig),
|
|
112
|
+
}, callVision, { instruction: typeof params.instruction === 'string' ? params.instruction : '', pageId }, signal);
|
|
113
|
+
return { ...result, details: undefined };
|
|
114
|
+
},
|
|
115
|
+
});
|
|
116
|
+
}
|
|
117
|
+
pi.on('session_start', async (_event, ctx) => {
|
|
118
|
+
config = resolveConfig(loadConfig({ cwd: ctx.cwd, projectTrusted: isProjectTrusted(ctx) }));
|
|
119
|
+
prepareBrowserProfile(config);
|
|
120
|
+
client = new DevToolsClient(config);
|
|
121
|
+
await registerUpstreamTools();
|
|
122
|
+
if (config.visionModel) {
|
|
123
|
+
await registerVisionTool(config.visionModel);
|
|
124
|
+
}
|
|
125
|
+
});
|
|
126
|
+
pi.on('session_shutdown', async () => {
|
|
127
|
+
if (client) {
|
|
128
|
+
await client.close();
|
|
129
|
+
client = undefined;
|
|
130
|
+
}
|
|
131
|
+
});
|
|
132
|
+
}
|
|
133
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAgB,MAAM,SAAS,CAAA;AAC5C,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAyB,MAAM,aAAa,CAAA;AAClE,OAAO,EAAE,gBAAgB,EAAE,UAAU,EAAE,MAAM,eAAe,CAAA;AAC5D,OAAO,EACL,sBAAsB,EACtB,kBAAkB,EAClB,qBAAqB,GACtB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAA;AACpD,OAAO,EACL,0BAA0B,EAC1B,uBAAuB,GAExB,MAAM,aAAa,CAAA;AAEpB,OAAO,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAEzD,gFAAgF;AAChF,MAAM,WAAW,GAAG,UAAU,CAAA;AAE9B,0EAA0E;AAC1E,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,kBAAkB;IAClB,6BAA6B;IAC7B,yBAAyB;IACzB,wBAAwB;IACxB,kBAAkB;IAClB,iBAAiB;IACjB,mBAAmB;IACnB,iBAAiB;IACjB,kBAAkB;IAClB,0BAA0B;IAC1B,qBAAqB;CACtB,CAAC,CAAA;AAuCF,SAAS,aAAa,CACpB,MAGC,EACD,YAAoB;IAEpB,MAAM,WAAW,GAAG,kBAAkB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;IACtD,MAAM,SAAS,GAAG,qBAAqB,CAAC,YAAY,EAAE,WAAW,CAAC,CAAA;IAClE,MAAM,OAAO,GAA6E,EAAE,CAAA;IAC5F,IAAI,SAAS,KAAK,WAAW,EAAE,CAAC;QAC9B,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC,CAAA;IACjD,CAAC;SAAM,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,IAAI,CAAC,IAAI;gBAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,CAAC,CAAA;QACxF,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;QACnB,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,OAAO,EAAE,CAAC;YAClC,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBACvC,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,IAAI,WAAW,EAAE,CAAC,CAAA;YAC1F,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAA;IAClE,OAAO,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,IAAa,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAA;AAC3E,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,OAAO,UAAU,mBAAmB,CAAC,EAAM;IAChD,IAAI,MAAoC,CAAA;IACxC,IAAI,MAAkC,CAAA;IAEtC,KAAK,UAAU,eAAe,CAAC,MAAoB;QACjD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAA;QAChE,MAAM,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAA;IAClC,CAAC;IAED,KAAK,UAAU,qBAAqB;QAClC,MAAM,eAAe,EAAE,CAAA;QACvB,MAAM,aAAa,GAAG,MAAM,MAAO,CAAC,YAAY,EAAE,CAAA;QAClD,KAAK,MAAM,IAAI,IAAI,aAAa,EAAE,CAAC;YACjC,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAQ;YAC3C,MAAM,YAAY,GAAG,GAAG,WAAW,GAAG,IAAI,CAAC,IAAI,EAAE,CAAA;YACjD,MAAM,YAAY,GAAG,IAAI,CAAC,IAAI,CAAA;YAC9B,MAAM,WAAW,GAAG,sBAAsB,CAAC,YAAY,EAAE,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;YAChF,EAAE,CAAC,YAAY,CAAC;gBACd,IAAI,EAAE,YAAY;gBAClB,KAAK,EAAE,YAAY;gBACnB,WAAW;gBACX,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,WAAsB,CAAC;gBACpD,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM;oBACvC,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;oBAC7B,MAAM,MAAM,GAAG,CAAC,MAAM,MAAO,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,MAAM,CAAC,CAGnE,CAAA;oBACD,OAAO,EAAE,GAAG,aAAa,CAAC,MAAM,EAAE,YAAY,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,CAAA;gBACvE,CAAC;aACF,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,KAAK,UAAU,kBAAkB,CAAC,YAA+B;QAC/D,MAAM,UAAU,GAA4B;YAC1C,WAAW,EAAE,IAAI,CAAC,QAAQ,CACxB,IAAI,CAAC,MAAM,CAAC;gBACV,WAAW,EACT,+FAA+F;aAClG,CAAC,CACH;SACF,CAAA;QACD,IAAI,MAAM,EAAE,yBAAyB,KAAK,IAAI,EAAE,CAAC;YAC/C,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;gBAC9B,WAAW,EAAE,iDAAiD;aAC/D,CAAC,CAAA;QACJ,CAAC;QACD,EAAE,CAAC,YAAY,CAAC;YACd,IAAI,EAAE,GAAG,WAAW,oBAAoB;YACxC,KAAK,EAAE,GAAG,WAAW,oBAAoB;YACzC,WAAW,EACT,6PAA6P;YAC/P,UAAU,EAAE,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC;YACnC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,GAAG;gBACvD,MAAM,eAAe,CAAC,MAAM,CAAC,CAAA;gBAC7B,IAAI,CAAC,GAAG,EAAE,aAAa;oBACrB,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAA;gBAC1E,MAAM,UAAU,GAAG,0BAA0B,CAAC,YAAY,EAAE,GAAG,CAAC,aAAa,CAAC,CAAA;gBAC9E,MAAM,MAAM,GAAG,OAAO,MAAM,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAA;gBAC5E,MAAM,MAAM,GAAG,MAAM,uBAAuB,CAC1C;oBACE,QAAQ,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE,CAAC,MAAO,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,EAAE,GAAG,CAAC;iBACjE,EACD,UAAU,EACV,EAAE,WAAW,EAAE,OAAO,MAAM,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,EAAE,MAAM,EAAE,EACzF,MAAM,CACP,CAAA;gBACD,OAAO,EAAE,GAAG,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,CAAA;YAC1C,CAAC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,EAAE,CAAC,EAAE,CAAC,eAAe,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE;QAC3C,MAAM,GAAG,aAAa,CAAC,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,cAAc,EAAE,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAA;QAC3F,qBAAqB,CAAC,MAAM,CAAC,CAAA;QAC7B,MAAM,GAAG,IAAI,cAAc,CAAC,MAAM,CAAC,CAAA;QACnC,MAAM,qBAAqB,EAAE,CAAA;QAC7B,IAAI,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,MAAM,kBAAkB,CAAC,MAAM,CAAC,WAAW,CAAC,CAAA;QAC9C,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,EAAE,CAAC,kBAAkB,EAAE,KAAK,IAAI,EAAE;QACnC,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,MAAM,CAAC,KAAK,EAAE,CAAA;YACpB,MAAM,GAAG,SAAS,CAAA;QACpB,CAAC;IACH,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { type BrowserUseConfig } from './config.js';
|
|
2
|
+
/**
|
|
3
|
+
* Guard the browser profile before launch. A default profile that exists but
|
|
4
|
+
* cannot be read/written (typically root-owned after running under sudo) is
|
|
5
|
+
* moved aside so Chrome starts fresh instead of showing a "can't read your
|
|
6
|
+
* preferences" dialog on every launch. An explicit custom userDataDir in the
|
|
7
|
+
* same state fails fast with a remediation hint instead of silently
|
|
8
|
+
* discarding the user's data.
|
|
9
|
+
*/
|
|
10
|
+
export declare function prepareBrowserProfile(config: BrowserUseConfig): void;
|
|
11
|
+
//# sourceMappingURL=profile.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"profile.d.ts","sourceRoot":"","sources":["../src/profile.ts"],"names":[],"mappings":"AAEA,OAAO,EAAuB,KAAK,gBAAgB,EAAE,MAAM,aAAa,CAAA;AAwBxE;;;;;;;GAOG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,gBAAgB,GAAG,IAAI,CA2BpE"}
|
package/dist/profile.js
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { accessSync, constants, renameSync, statSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { DEFAULT_PROFILE_DIR } from './config.js';
|
|
4
|
+
function isRootOwned(path) {
|
|
5
|
+
try {
|
|
6
|
+
const stat = statSync(path);
|
|
7
|
+
const getuid = process.getuid;
|
|
8
|
+
return stat.uid === 0 && typeof getuid === 'function' && getuid() !== 0;
|
|
9
|
+
}
|
|
10
|
+
catch {
|
|
11
|
+
return false;
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
function isUnusableDir(path) {
|
|
15
|
+
try {
|
|
16
|
+
if (isRootOwned(path))
|
|
17
|
+
return true;
|
|
18
|
+
accessSync(path, constants.R_OK | constants.W_OK);
|
|
19
|
+
return false;
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
// A missing directory is fine (Chrome creates it); anything else means
|
|
23
|
+
// the profile cannot be used.
|
|
24
|
+
return error.code !== 'ENOENT';
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Guard the browser profile before launch. A default profile that exists but
|
|
29
|
+
* cannot be read/written (typically root-owned after running under sudo) is
|
|
30
|
+
* moved aside so Chrome starts fresh instead of showing a "can't read your
|
|
31
|
+
* preferences" dialog on every launch. An explicit custom userDataDir in the
|
|
32
|
+
* same state fails fast with a remediation hint instead of silently
|
|
33
|
+
* discarding the user's data.
|
|
34
|
+
*/
|
|
35
|
+
export function prepareBrowserProfile(config) {
|
|
36
|
+
if (config.browserUrl || config.wsEndpoint)
|
|
37
|
+
return;
|
|
38
|
+
const dir = config.userDataDir ?? (config.sessionMode === 'persistent' ? DEFAULT_PROFILE_DIR : undefined);
|
|
39
|
+
if (!dir || dir !== DEFAULT_PROFILE_DIR) {
|
|
40
|
+
if (dir && isUnusableDir(dir)) {
|
|
41
|
+
throw new Error(`Browser profile at ${dir} is not accessible (likely root-owned from running under sudo). ` +
|
|
42
|
+
`chown it back with: sudo chown -R $(id -un):$(id -gn) ${JSON.stringify(dir)}`);
|
|
43
|
+
}
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
if (isUnusableDir(dir)) {
|
|
47
|
+
const aside = join(dirname(DEFAULT_PROFILE_DIR), `browser-profile.inaccessible-${Date.now()}`);
|
|
48
|
+
console.error(`[pi-browser-use] default browser profile is inaccessible; moving it aside to ${aside} and starting fresh.`);
|
|
49
|
+
try {
|
|
50
|
+
renameSync(dir, aside);
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
throw new Error(`Browser profile at ${dir} is not accessible and could not be moved aside. ` +
|
|
54
|
+
`Fix ownership with: sudo chown -R $(id -un):$(id -gn) ${JSON.stringify(dir)}`);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
//# sourceMappingURL=profile.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"profile.js","sourceRoot":"","sources":["../src/profile.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAA;AACrE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AACzC,OAAO,EAAE,mBAAmB,EAAyB,MAAM,aAAa,CAAA;AAExE,SAAS,WAAW,CAAC,IAAY;IAC/B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAA;QAC3B,MAAM,MAAM,GAAI,OAAgD,CAAC,MAAM,CAAA;QACvE,OAAO,IAAI,CAAC,GAAG,KAAK,CAAC,IAAI,OAAO,MAAM,KAAK,UAAU,IAAI,MAAM,EAAE,KAAK,CAAC,CAAA;IACzE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC;AAED,SAAS,aAAa,CAAC,IAAY;IACjC,IAAI,CAAC;QACH,IAAI,WAAW,CAAC,IAAI,CAAC;YAAE,OAAO,IAAI,CAAA;QAClC,UAAU,CAAC,IAAI,EAAE,SAAS,CAAC,IAAI,GAAG,SAAS,CAAC,IAAI,CAAC,CAAA;QACjD,OAAO,KAAK,CAAA;IACd,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,uEAAuE;QACvE,8BAA8B;QAC9B,OAAQ,KAA+B,CAAC,IAAI,KAAK,QAAQ,CAAA;IAC3D,CAAC;AACH,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,UAAU,qBAAqB,CAAC,MAAwB;IAC5D,IAAI,MAAM,CAAC,UAAU,IAAI,MAAM,CAAC,UAAU;QAAE,OAAM;IAClD,MAAM,GAAG,GACP,MAAM,CAAC,WAAW,IAAI,CAAC,MAAM,CAAC,WAAW,KAAK,YAAY,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,SAAS,CAAC,CAAA;IAC/F,IAAI,CAAC,GAAG,IAAI,GAAG,KAAK,mBAAmB,EAAE,CAAC;QACxC,IAAI,GAAG,IAAI,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;YAC9B,MAAM,IAAI,KAAK,CACb,sBAAsB,GAAG,kEAAkE;gBACzF,yDAAyD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CACjF,CAAA;QACH,CAAC;QACD,OAAM;IACR,CAAC;IACD,IAAI,aAAa,CAAC,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,KAAK,GAAG,IAAI,CAAC,OAAO,CAAC,mBAAmB,CAAC,EAAE,gCAAgC,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC,CAAA;QAC9F,OAAO,CAAC,KAAK,CACX,gFAAgF,KAAK,sBAAsB,CAC5G,CAAA;QACD,IAAI,CAAC;YACH,UAAU,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACxB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,MAAM,IAAI,KAAK,CACb,sBAAsB,GAAG,mDAAmD;gBAC1E,yDAAyD,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CACjF,CAAA;QACH,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { BrowserUseConfig } from './config.js';
|
|
2
|
+
/**
|
|
3
|
+
* Load the "pi-browser-use" section. Priority: user settings, then trusted
|
|
4
|
+
* project settings (<cwd>/.pi/settings.json, no env expansion). Project
|
|
5
|
+
* settings apply only when the session trusts the project.
|
|
6
|
+
*/
|
|
7
|
+
export declare function loadConfig(options?: {
|
|
8
|
+
cwd?: string;
|
|
9
|
+
projectTrusted?: boolean;
|
|
10
|
+
}): BrowserUseConfig;
|
|
11
|
+
export declare function isProjectTrusted(context: unknown): boolean;
|
|
12
|
+
//# sourceMappingURL=settings.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"settings.d.ts","sourceRoot":"","sources":["../src/settings.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAA;AA4BnD;;;;GAIG;AACH,wBAAgB,UAAU,CAAC,OAAO,CAAC,EAAE;IAAE,GAAG,CAAC,EAAE,MAAM,CAAC;IAAC,cAAc,CAAC,EAAE,OAAO,CAAA;CAAE,GAAG,gBAAgB,CAMjG;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,OAAO,GAAG,OAAO,CAO1D"}
|
package/dist/settings.js
ADDED
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { homedir } from 'node:os';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
function expandEnv(value) {
|
|
5
|
+
if (typeof value === 'string') {
|
|
6
|
+
return value.replace(/\$\{([^}]+)\}/g, (_m, expr) => {
|
|
7
|
+
const [name, ...rest] = expr.split(':-');
|
|
8
|
+
return process.env[name] ?? rest.join(':-');
|
|
9
|
+
});
|
|
10
|
+
}
|
|
11
|
+
if (Array.isArray(value))
|
|
12
|
+
return value.map(expandEnv);
|
|
13
|
+
if (value && typeof value === 'object') {
|
|
14
|
+
return Object.fromEntries(Object.entries(value).map(([k, v]) => [k, expandEnv(v)]));
|
|
15
|
+
}
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
function readSection(filePath, expand) {
|
|
19
|
+
try {
|
|
20
|
+
const parsed = JSON.parse(readFileSync(filePath, 'utf-8'));
|
|
21
|
+
const section = parsed['pi-browser-use'];
|
|
22
|
+
if (!section || typeof section !== 'object' || Array.isArray(section))
|
|
23
|
+
return {};
|
|
24
|
+
return (expand ? expandEnv(section) : section);
|
|
25
|
+
}
|
|
26
|
+
catch (error) {
|
|
27
|
+
if (error.code === 'ENOENT')
|
|
28
|
+
return {};
|
|
29
|
+
throw error;
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Load the "pi-browser-use" section. Priority: user settings, then trusted
|
|
34
|
+
* project settings (<cwd>/.pi/settings.json, no env expansion). Project
|
|
35
|
+
* settings apply only when the session trusts the project.
|
|
36
|
+
*/
|
|
37
|
+
export function loadConfig(options) {
|
|
38
|
+
const user = readSection(join(homedir(), '.pi', 'agent', 'settings.json'), true);
|
|
39
|
+
const cwd = options?.cwd ?? process.cwd();
|
|
40
|
+
const project = options?.projectTrusted === true ? readSection(join(cwd, '.pi', 'settings.json'), false) : {};
|
|
41
|
+
return { ...user, ...project };
|
|
42
|
+
}
|
|
43
|
+
export function isProjectTrusted(context) {
|
|
44
|
+
const ctx = context;
|
|
45
|
+
try {
|
|
46
|
+
return ctx?.isProjectTrusted?.() === true;
|
|
47
|
+
}
|
|
48
|
+
catch {
|
|
49
|
+
return false;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=settings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"settings.js","sourceRoot":"","sources":["../src/settings.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAA;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,SAAS,CAAA;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAGhC,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,KAAK,CAAC,OAAO,CAAC,gBAAgB,EAAE,CAAC,EAAE,EAAE,IAAY,EAAE,EAAE;YAC1D,MAAM,CAAC,IAAI,EAAE,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YACxC,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QAC7C,CAAC,CAAC,CAAA;IACJ,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAA;IACrD,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QACvC,OAAO,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAA;IACrF,CAAC;IACD,OAAO,KAAK,CAAA;AACd,CAAC;AAED,SAAS,WAAW,CAAC,QAAgB,EAAE,MAAe;IACpD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,EAAE,OAAO,CAAC,CAA4B,CAAA;QACrF,MAAM,OAAO,GAAG,MAAM,CAAC,gBAAgB,CAAC,CAAA;QACxC,IAAI,CAAC,OAAO,IAAI,OAAO,OAAO,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;YAAE,OAAO,EAAE,CAAA;QAChF,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,OAAO,CAAqB,CAAA;IACpE,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;YAAE,OAAO,EAAE,CAAA;QACjE,MAAM,KAAK,CAAA;IACb,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,UAAU,CAAC,OAAoD;IAC7E,MAAM,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,eAAe,CAAC,EAAE,IAAI,CAAC,CAAA;IAChF,MAAM,GAAG,GAAG,OAAO,EAAE,GAAG,IAAI,OAAO,CAAC,GAAG,EAAE,CAAA;IACzC,MAAM,OAAO,GACX,OAAO,EAAE,cAAc,KAAK,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,eAAe,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAA;IAC/F,OAAO,EAAE,GAAG,IAAI,EAAE,GAAG,OAAO,EAAE,CAAA;AAChC,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,OAAgB;IAC/C,MAAM,GAAG,GAAG,OAAkE,CAAA;IAC9E,IAAI,CAAC;QACH,OAAO,GAAG,EAAE,gBAAgB,EAAE,EAAE,KAAK,IAAI,CAAA;IAC3C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
export declare function augmentToolDescription(prefixedName: string, description: string): string;
|
|
2
|
+
export declare function postProcessToolResult(originalName: string, text: string): string;
|
|
3
|
+
export declare function extractTextContent(content: unknown): string;
|
|
4
|
+
//# sourceMappingURL=tool-augment.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-augment.d.ts","sourceRoot":"","sources":["../src/tool-augment.ts"],"names":[],"mappings":"AASA,wBAAgB,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,GAAG,MAAM,CAGxF;AAED,wBAAgB,qBAAqB,CAAC,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,MAAM,CAQhF;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,OAAO,GAAG,MAAM,CAS3D"}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
const HINTS = {
|
|
2
|
+
browser_click: 'Use the element uid from the accessibility tree snapshot; UIDs are invalidated after this action.',
|
|
3
|
+
browser_fill: 'Fills standard HTML form fields only; does not work on canvas/custom widgets.',
|
|
4
|
+
browser_press_key: 'Accepts a single key name only (e.g. Enter, Tab, Escape).',
|
|
5
|
+
browser_take_snapshot: 'Call first to get uids, and after every state-changing action.',
|
|
6
|
+
browser_navigate_page: 'Call take_snapshot after navigation to see the new page.',
|
|
7
|
+
};
|
|
8
|
+
export function augmentToolDescription(prefixedName, description) {
|
|
9
|
+
const hint = HINTS[prefixedName];
|
|
10
|
+
return hint ? `${description}\n\nHint: ${hint}` : description;
|
|
11
|
+
}
|
|
12
|
+
export function postProcessToolResult(originalName, text) {
|
|
13
|
+
if (originalName === 'click' && /overlay|obscured|intercept/i.test(text)) {
|
|
14
|
+
return `${text}\n\nHint: the click was blocked by an overlay/popup — dismiss it first, then refresh the snapshot.`;
|
|
15
|
+
}
|
|
16
|
+
if (/stale|no longer attached|not found/i.test(text) && /uid|element/i.test(text)) {
|
|
17
|
+
return `${text}\n\nHint: element references are stale — take a fresh snapshot for current uids.`;
|
|
18
|
+
}
|
|
19
|
+
return text;
|
|
20
|
+
}
|
|
21
|
+
export function extractTextContent(content) {
|
|
22
|
+
if (!Array.isArray(content))
|
|
23
|
+
return '';
|
|
24
|
+
return content
|
|
25
|
+
.filter((item) => typeof item === 'object' && item !== null)
|
|
26
|
+
.filter((item) => item.type === 'text' && typeof item.text === 'string')
|
|
27
|
+
.map((item) => item.text)
|
|
28
|
+
.join('\n');
|
|
29
|
+
}
|
|
30
|
+
//# sourceMappingURL=tool-augment.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"tool-augment.js","sourceRoot":"","sources":["../src/tool-augment.ts"],"names":[],"mappings":"AAAA,MAAM,KAAK,GAA2B;IACpC,aAAa,EACX,mGAAmG;IACrG,YAAY,EAAE,+EAA+E;IAC7F,iBAAiB,EAAE,2DAA2D;IAC9E,qBAAqB,EAAE,gEAAgE;IACvF,qBAAqB,EAAE,0DAA0D;CAClF,CAAA;AAED,MAAM,UAAU,sBAAsB,CAAC,YAAoB,EAAE,WAAmB;IAC9E,MAAM,IAAI,GAAG,KAAK,CAAC,YAAY,CAAC,CAAA;IAChC,OAAO,IAAI,CAAC,CAAC,CAAC,GAAG,WAAW,aAAa,IAAI,EAAE,CAAC,CAAC,CAAC,WAAW,CAAA;AAC/D,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,YAAoB,EAAE,IAAY;IACtE,IAAI,YAAY,KAAK,OAAO,IAAI,6BAA6B,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QACzE,OAAO,GAAG,IAAI,oGAAoG,CAAA;IACpH,CAAC;IACD,IAAI,qCAAqC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,cAAc,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;QAClF,OAAO,GAAG,IAAI,kFAAkF,CAAA;IAClG,CAAC;IACD,OAAO,IAAI,CAAA;AACb,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,OAAgB;IACjD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC;QAAE,OAAO,EAAE,CAAA;IACtC,OAAO,OAAO;SACX,MAAM,CACL,CAAC,IAAI,EAA2C,EAAE,CAAC,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,CAC7F;SACA,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;SACvE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAc,CAAC;SAClC,IAAI,CAAC,IAAI,CAAC,CAAA;AACf,CAAC"}
|
package/dist/vision.d.ts
ADDED
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Optional vision-model integration.
|
|
3
|
+
*
|
|
4
|
+
* Captures a screenshot via the upstream take_screenshot tool and sends it
|
|
5
|
+
* to a vision model from Pi's model registry. This lets the agent identify
|
|
6
|
+
* elements by visual attributes (color, layout, coordinates) when the
|
|
7
|
+
* accessibility tree is insufficient, e.g. canvas/WebGL scenes.
|
|
8
|
+
*
|
|
9
|
+
* The pi-ai host module is loaded lazily so the extension runs on hosts
|
|
10
|
+
* without it; only the vision tool requires it.
|
|
11
|
+
*/
|
|
12
|
+
export interface VisionModelConfig {
|
|
13
|
+
provider: string;
|
|
14
|
+
model: string;
|
|
15
|
+
}
|
|
16
|
+
export declare const VISUAL_SYSTEM_PROMPT = "You are a visual analysis assistant for browser automation. You receive a screenshot of a web page and an instruction.\n\nCOORDINATES:\n- Pixel-based, relative to the visible viewport; (0, 0) is the top-left.\n- Estimate positions from the screenshot.\n\nRULES:\n- You only analyze; you never act.\n- Give exact (x, y) coordinates when the instruction asks for an element, so the caller can use coordinate click tools.\n- If the target is not visible, say so explicitly instead of guessing.\n- Be concise and actionable.";
|
|
17
|
+
interface ModelRegistry {
|
|
18
|
+
find(provider: string, modelId: string): unknown;
|
|
19
|
+
getApiKeyAndHeaders(model: unknown): Promise<{
|
|
20
|
+
ok: true;
|
|
21
|
+
apiKey?: string;
|
|
22
|
+
headers?: Record<string, string>;
|
|
23
|
+
} | {
|
|
24
|
+
ok: false;
|
|
25
|
+
error: string;
|
|
26
|
+
}>;
|
|
27
|
+
}
|
|
28
|
+
export declare function createRegistryVisionCaller(visionConfig: VisionModelConfig, registry: ModelRegistry): (instruction: string, imageBase64: string, mimeType: string, signal?: AbortSignal) => Promise<string>;
|
|
29
|
+
interface BrowserClient {
|
|
30
|
+
callTool(name: string, args: unknown, signal?: AbortSignal): Promise<unknown>;
|
|
31
|
+
}
|
|
32
|
+
/**
|
|
33
|
+
* Capture a screenshot through the browser client and analyze it with the
|
|
34
|
+
* provided vision caller. Returns MCP-style content, never throws except on
|
|
35
|
+
* abort: failures degrade to actionable error text pointing at the
|
|
36
|
+
* accessibility tree instead.
|
|
37
|
+
*/
|
|
38
|
+
export declare function handleAnalyzeScreenshot(client: BrowserClient, callVision: (instruction: string, imageBase64: string, mimeType: string, signal?: AbortSignal) => Promise<string>, args: {
|
|
39
|
+
instruction?: string;
|
|
40
|
+
pageId?: number;
|
|
41
|
+
}, signal?: AbortSignal): Promise<{
|
|
42
|
+
content: Array<{
|
|
43
|
+
type: string;
|
|
44
|
+
text?: string;
|
|
45
|
+
}>;
|
|
46
|
+
isError?: boolean;
|
|
47
|
+
}>;
|
|
48
|
+
export {};
|
|
49
|
+
//# sourceMappingURL=vision.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vision.d.ts","sourceRoot":"","sources":["../src/vision.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,MAAM,WAAW,iBAAiB;IAChC,QAAQ,EAAE,MAAM,CAAA;IAChB,KAAK,EAAE,MAAM,CAAA;CACd;AAED,eAAO,MAAM,oBAAoB,6gBAUJ,CAAA;AA4B7B,UAAU,aAAa;IACrB,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,GAAG,OAAO,CAAA;IAChD,mBAAmB,CACjB,KAAK,EAAE,OAAO,GACb,OAAO,CACR;QAAE,EAAE,EAAE,IAAI,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;KAAE,GAAG;QAAE,EAAE,EAAE,KAAK,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAC/F,CAAA;CACF;AAUD,wBAAgB,0BAA0B,CACxC,YAAY,EAAE,iBAAiB,EAC/B,QAAQ,EAAE,aAAa,GACtB,CACD,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,KACjB,OAAO,CAAC,MAAM,CAAC,CA6CnB;AAOD,UAAU,aAAa;IACrB,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,OAAO,CAAC,CAAA;CAC9E;AAED;;;;;GAKG;AACH,wBAAsB,uBAAuB,CAC3C,MAAM,EAAE,aAAa,EACrB,UAAU,EAAE,CACV,WAAW,EAAE,MAAM,EACnB,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,MAAM,CAAC,EAAE,WAAW,KACjB,OAAO,CAAC,MAAM,CAAC,EACpB,IAAI,EAAE;IAAE,WAAW,CAAC,EAAE,MAAM,CAAC;IAAC,MAAM,CAAC,EAAE,MAAM,CAAA;CAAE,EAC/C,MAAM,CAAC,EAAE,WAAW,GACnB,OAAO,CAAC;IAAE,OAAO,EAAE,KAAK,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,IAAI,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAAC,OAAO,CAAC,EAAE,OAAO,CAAA;CAAE,CAAC,CAqEjF"}
|
package/dist/vision.js
ADDED
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Optional vision-model integration.
|
|
3
|
+
*
|
|
4
|
+
* Captures a screenshot via the upstream take_screenshot tool and sends it
|
|
5
|
+
* to a vision model from Pi's model registry. This lets the agent identify
|
|
6
|
+
* elements by visual attributes (color, layout, coordinates) when the
|
|
7
|
+
* accessibility tree is insufficient, e.g. canvas/WebGL scenes.
|
|
8
|
+
*
|
|
9
|
+
* The pi-ai host module is loaded lazily so the extension runs on hosts
|
|
10
|
+
* without it; only the vision tool requires it.
|
|
11
|
+
*/
|
|
12
|
+
export const VISUAL_SYSTEM_PROMPT = `You are a visual analysis assistant for browser automation. You receive a screenshot of a web page and an instruction.
|
|
13
|
+
|
|
14
|
+
COORDINATES:
|
|
15
|
+
- Pixel-based, relative to the visible viewport; (0, 0) is the top-left.
|
|
16
|
+
- Estimate positions from the screenshot.
|
|
17
|
+
|
|
18
|
+
RULES:
|
|
19
|
+
- You only analyze; you never act.
|
|
20
|
+
- Give exact (x, y) coordinates when the instruction asks for an element, so the caller can use coordinate click tools.
|
|
21
|
+
- If the target is not visible, say so explicitly instead of guessing.
|
|
22
|
+
- Be concise and actionable.`;
|
|
23
|
+
async function loadComplete() {
|
|
24
|
+
const mod = (await import('@earendil-works/pi-ai/compat'));
|
|
25
|
+
if (typeof mod.complete !== 'function') {
|
|
26
|
+
throw new Error('Host pi-ai does not expose compat.complete; visual analysis is unavailable.');
|
|
27
|
+
}
|
|
28
|
+
return mod.complete;
|
|
29
|
+
}
|
|
30
|
+
export function createRegistryVisionCaller(visionConfig, registry) {
|
|
31
|
+
return async (instruction, imageBase64, mimeType, signal) => {
|
|
32
|
+
const model = registry.find(visionConfig.provider, visionConfig.model);
|
|
33
|
+
if (!model) {
|
|
34
|
+
throw new Error(`Vision model "${visionConfig.provider}/${visionConfig.model}" not found in the model registry.`);
|
|
35
|
+
}
|
|
36
|
+
const auth = await registry.getApiKeyAndHeaders(model);
|
|
37
|
+
if (!auth.ok) {
|
|
38
|
+
throw new Error(`Auth failed for vision model: ${auth.error}`);
|
|
39
|
+
}
|
|
40
|
+
const options = { maxTokens: 2048 };
|
|
41
|
+
if (auth.apiKey)
|
|
42
|
+
options.apiKey = auth.apiKey;
|
|
43
|
+
if (auth.headers)
|
|
44
|
+
options.headers = auth.headers;
|
|
45
|
+
if (signal)
|
|
46
|
+
options.signal = signal;
|
|
47
|
+
const complete = await loadComplete();
|
|
48
|
+
const result = await complete(model, {
|
|
49
|
+
systemPrompt: VISUAL_SYSTEM_PROMPT,
|
|
50
|
+
messages: [
|
|
51
|
+
{
|
|
52
|
+
role: 'user',
|
|
53
|
+
content: [
|
|
54
|
+
{
|
|
55
|
+
type: 'text',
|
|
56
|
+
text: `Analyze this screenshot and respond to the following instruction:\n\n${instruction}`,
|
|
57
|
+
},
|
|
58
|
+
{ type: 'image', data: imageBase64, mimeType },
|
|
59
|
+
],
|
|
60
|
+
timestamp: Date.now(),
|
|
61
|
+
},
|
|
62
|
+
],
|
|
63
|
+
}, options);
|
|
64
|
+
if (result.stopReason === 'error') {
|
|
65
|
+
throw new Error(result.errorMessage || 'Vision model request failed');
|
|
66
|
+
}
|
|
67
|
+
return result.content
|
|
68
|
+
.filter((part) => part.type === 'text' && typeof part.text === 'string')
|
|
69
|
+
.map((part) => part.text)
|
|
70
|
+
.join('');
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/**
|
|
74
|
+
* Capture a screenshot through the browser client and analyze it with the
|
|
75
|
+
* provided vision caller. Returns MCP-style content, never throws except on
|
|
76
|
+
* abort: failures degrade to actionable error text pointing at the
|
|
77
|
+
* accessibility tree instead.
|
|
78
|
+
*/
|
|
79
|
+
export async function handleAnalyzeScreenshot(client, callVision, args, signal) {
|
|
80
|
+
const instruction = String(args.instruction ?? '');
|
|
81
|
+
const screenshotArgs = typeof args.pageId === 'number' ? { pageId: args.pageId } : {};
|
|
82
|
+
try {
|
|
83
|
+
const screenshotResult = (await client.callTool('take_screenshot', screenshotArgs, signal));
|
|
84
|
+
let imageBase64 = '';
|
|
85
|
+
let mimeType = 'image/png';
|
|
86
|
+
if (Array.isArray(screenshotResult.content)) {
|
|
87
|
+
for (const item of screenshotResult.content) {
|
|
88
|
+
if (item.type === 'image' && item.data) {
|
|
89
|
+
imageBase64 = item.data;
|
|
90
|
+
mimeType = item.mimeType ?? 'image/png';
|
|
91
|
+
break;
|
|
92
|
+
}
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
if (!imageBase64) {
|
|
96
|
+
return {
|
|
97
|
+
content: [
|
|
98
|
+
{
|
|
99
|
+
type: 'text',
|
|
100
|
+
text: 'Failed to capture a screenshot for visual analysis. Use accessibility tree elements instead.',
|
|
101
|
+
},
|
|
102
|
+
],
|
|
103
|
+
isError: true,
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
const analysis = await callVision(instruction, imageBase64, mimeType, signal);
|
|
107
|
+
if (!analysis) {
|
|
108
|
+
return {
|
|
109
|
+
content: [
|
|
110
|
+
{
|
|
111
|
+
type: 'text',
|
|
112
|
+
text: 'The visual model returned no analysis. Use accessibility tree elements instead.',
|
|
113
|
+
},
|
|
114
|
+
],
|
|
115
|
+
isError: true,
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
return { content: [{ type: 'text', text: `Visual analysis result:\n${analysis}` }] };
|
|
119
|
+
}
|
|
120
|
+
catch (error) {
|
|
121
|
+
if (signal?.aborted)
|
|
122
|
+
throw error;
|
|
123
|
+
const message = error instanceof Error ? error.message : String(error);
|
|
124
|
+
const unavailable = message.includes('404') ||
|
|
125
|
+
message.includes('403') ||
|
|
126
|
+
message.includes('not found') ||
|
|
127
|
+
message.includes('permission');
|
|
128
|
+
if (!unavailable) {
|
|
129
|
+
console.error(`[pi-browser-use] visual analysis failed (${error instanceof Error ? error.name : 'UnknownError'})`);
|
|
130
|
+
}
|
|
131
|
+
return {
|
|
132
|
+
content: [
|
|
133
|
+
{
|
|
134
|
+
type: 'text',
|
|
135
|
+
text: unavailable
|
|
136
|
+
? 'The visual analysis model is unavailable. Use accessibility tree elements (uids from take_snapshot) for all interactions instead.'
|
|
137
|
+
: 'Visual analysis failed. Use accessibility tree elements instead.',
|
|
138
|
+
},
|
|
139
|
+
],
|
|
140
|
+
isError: true,
|
|
141
|
+
};
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
//# sourceMappingURL=vision.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vision.js","sourceRoot":"","sources":["../src/vision.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAOH,MAAM,CAAC,MAAM,oBAAoB,GAAG;;;;;;;;;;6BAUP,CAAA;AAqC7B,KAAK,UAAU,YAAY;IACzB,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,8BAA8B,CAAC,CAA8B,CAAA;IACvF,IAAI,OAAO,GAAG,CAAC,QAAQ,KAAK,UAAU,EAAE,CAAC;QACvC,MAAM,IAAI,KAAK,CAAC,6EAA6E,CAAC,CAAA;IAChG,CAAC;IACD,OAAO,GAAG,CAAC,QAAQ,CAAA;AACrB,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,YAA+B,EAC/B,QAAuB;IAOvB,OAAO,KAAK,EAAE,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE;QAC1D,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,YAAY,CAAC,QAAQ,EAAE,YAAY,CAAC,KAAK,CAAC,CAAA;QACtE,IAAI,CAAC,KAAK,EAAE,CAAC;YACX,MAAM,IAAI,KAAK,CACb,iBAAiB,YAAY,CAAC,QAAQ,IAAI,YAAY,CAAC,KAAK,oCAAoC,CACjG,CAAA;QACH,CAAC;QACD,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,mBAAmB,CAAC,KAAK,CAAC,CAAA;QACtD,IAAI,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;YACb,MAAM,IAAI,KAAK,CAAC,iCAAiC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAA;QAChE,CAAC;QACD,MAAM,OAAO,GAA4B,EAAE,SAAS,EAAE,IAAI,EAAE,CAAA;QAC5D,IAAI,IAAI,CAAC,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QAC7C,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAA;QAChD,IAAI,MAAM;YAAE,OAAO,CAAC,MAAM,GAAG,MAAM,CAAA;QACnC,MAAM,QAAQ,GAAG,MAAM,YAAY,EAAE,CAAA;QACrC,MAAM,MAAM,GAAG,MAAM,QAAQ,CAC3B,KAAK,EACL;YACE,YAAY,EAAE,oBAAoB;YAClC,QAAQ,EAAE;gBACR;oBACE,IAAI,EAAE,MAAM;oBACZ,OAAO,EAAE;wBACP;4BACE,IAAI,EAAE,MAAM;4BACZ,IAAI,EAAE,wEAAwE,WAAW,EAAE;yBAC5F;wBACD,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,WAAW,EAAE,QAAQ,EAAE;qBAC/C;oBACD,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;iBACtB;aACF;SACF,EACD,OAAO,CACR,CAAA;QACD,IAAI,MAAM,CAAC,UAAU,KAAK,OAAO,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC,YAAY,IAAI,6BAA6B,CAAC,CAAA;QACvE,CAAC;QACD,OAAO,MAAM,CAAC,OAAO;aAClB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,IAAI,OAAO,IAAI,CAAC,IAAI,KAAK,QAAQ,CAAC;aACvE,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAc,CAAC;aAClC,IAAI,CAAC,EAAE,CAAC,CAAA;IACb,CAAC,CAAA;AACH,CAAC;AAWD;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,MAAqB,EACrB,UAKoB,EACpB,IAA+C,EAC/C,MAAoB;IAEpB,MAAM,WAAW,GAAG,MAAM,CAAC,IAAI,CAAC,WAAW,IAAI,EAAE,CAAC,CAAA;IAClD,MAAM,cAAc,GAAG,OAAO,IAAI,CAAC,MAAM,KAAK,QAAQ,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAA;IACrF,IAAI,CAAC;QACH,MAAM,gBAAgB,GAAG,CAAC,MAAM,MAAM,CAAC,QAAQ,CAC7C,iBAAiB,EACjB,cAAc,EACd,MAAM,CACP,CAAkB,CAAA;QACnB,IAAI,WAAW,GAAG,EAAE,CAAA;QACpB,IAAI,QAAQ,GAAG,WAAW,CAAA;QAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,gBAAgB,CAAC,OAAO,CAAC,EAAE,CAAC;YAC5C,KAAK,MAAM,IAAI,IAAI,gBAAgB,CAAC,OAAO,EAAE,CAAC;gBAC5C,IAAI,IAAI,CAAC,IAAI,KAAK,OAAO,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;oBACvC,WAAW,GAAG,IAAI,CAAC,IAAI,CAAA;oBACvB,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,WAAW,CAAA;oBACvC,MAAK;gBACP,CAAC;YACH,CAAC;QACH,CAAC;QACD,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,8FAA8F;qBACrG;iBACF;gBACD,OAAO,EAAE,IAAI;aACd,CAAA;QACH,CAAC;QACD,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,WAAW,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAA;QAC7E,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,OAAO;gBACL,OAAO,EAAE;oBACP;wBACE,IAAI,EAAE,MAAM;wBACZ,IAAI,EAAE,iFAAiF;qBACxF;iBACF;gBACD,OAAO,EAAE,IAAI;aACd,CAAA;QACH,CAAC;QACD,OAAO,EAAE,OAAO,EAAE,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,4BAA4B,QAAQ,EAAE,EAAE,CAAC,EAAE,CAAA;IACtF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,MAAM,EAAE,OAAO;YAAE,MAAM,KAAK,CAAA;QAChC,MAAM,OAAO,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;QACtE,MAAM,WAAW,GACf,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;YACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAC;YACvB,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAC;YAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAA;QAChC,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,OAAO,CAAC,KAAK,CACX,4CAA4C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,cAAc,GAAG,CACpG,CAAA;QACH,CAAC;QACD,OAAO;YACL,OAAO,EAAE;gBACP;oBACE,IAAI,EAAE,MAAM;oBACZ,IAAI,EAAE,WAAW;wBACf,CAAC,CAAC,mIAAmI;wBACrI,CAAC,CAAC,kEAAkE;iBACvE;aACF;YACD,OAAO,EAAE,IAAI;SACd,CAAA;IACH,CAAC;AACH,CAAC"}
|
package/package.json
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "pi-browser-use",
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Opinionated browser-use for the Pi coding agent, powered by chrome-devtools-mcp (not Playwright). Fresh headless by default, authenticated persistent profile opt-in, CLI-first policy bundled.",
|
|
5
|
+
"keywords": [
|
|
6
|
+
"pi-package",
|
|
7
|
+
"pi",
|
|
8
|
+
"extension",
|
|
9
|
+
"browser",
|
|
10
|
+
"automation",
|
|
11
|
+
"chrome-devtools",
|
|
12
|
+
"mcp"
|
|
13
|
+
],
|
|
14
|
+
"license": "MIT",
|
|
15
|
+
"author": "0xPlayerOne",
|
|
16
|
+
"repository": {
|
|
17
|
+
"type": "git",
|
|
18
|
+
"url": "https://github.com/0xPlayerOne/pi-browser-use.git"
|
|
19
|
+
},
|
|
20
|
+
"homepage": "https://github.com/0xPlayerOne/pi-browser-use#readme",
|
|
21
|
+
"bugs": {
|
|
22
|
+
"url": "https://github.com/0xPlayerOne/pi-browser-use/issues"
|
|
23
|
+
},
|
|
24
|
+
"type": "module",
|
|
25
|
+
"engines": {
|
|
26
|
+
"node": "^20.19.0 || ^22.12.0 || >=23"
|
|
27
|
+
},
|
|
28
|
+
"sideEffects": false,
|
|
29
|
+
"main": "./dist/index.js",
|
|
30
|
+
"types": "./dist/index.d.ts",
|
|
31
|
+
"exports": {
|
|
32
|
+
".": {
|
|
33
|
+
"types": "./dist/index.d.ts",
|
|
34
|
+
"default": "./dist/index.js"
|
|
35
|
+
},
|
|
36
|
+
"./package.json": {
|
|
37
|
+
"default": "./package.json"
|
|
38
|
+
}
|
|
39
|
+
},
|
|
40
|
+
"files": [
|
|
41
|
+
"dist",
|
|
42
|
+
"skills",
|
|
43
|
+
"README.md"
|
|
44
|
+
],
|
|
45
|
+
"pi": {
|
|
46
|
+
"extensions": [
|
|
47
|
+
"./dist/index.js"
|
|
48
|
+
],
|
|
49
|
+
"skills": [
|
|
50
|
+
"./skills"
|
|
51
|
+
]
|
|
52
|
+
},
|
|
53
|
+
"scripts": {
|
|
54
|
+
"build": "tsc -p tsconfig.json",
|
|
55
|
+
"format:check": "prettier --check .",
|
|
56
|
+
"typecheck": "tsc -p tsconfig.json --noEmit",
|
|
57
|
+
"prepublishOnly": "npm run build",
|
|
58
|
+
"test": "node --test test/*.test.mjs",
|
|
59
|
+
"pretest": "npm run build"
|
|
60
|
+
},
|
|
61
|
+
"dependencies": {
|
|
62
|
+
"@modelcontextprotocol/sdk": "^1.29.0",
|
|
63
|
+
"chrome-devtools-mcp": "1.8.0",
|
|
64
|
+
"typebox": "*"
|
|
65
|
+
},
|
|
66
|
+
"peerDependencies": {
|
|
67
|
+
"@earendil-works/pi-coding-agent": ">=0.80.10",
|
|
68
|
+
"@earendil-works/pi-ai": ">=0.80.10"
|
|
69
|
+
},
|
|
70
|
+
"devDependencies": {
|
|
71
|
+
"@earendil-works/pi-ai": "0.85.1",
|
|
72
|
+
"@types/node": "^22.0.0",
|
|
73
|
+
"prettier": "^3.9.6",
|
|
74
|
+
"typescript": "^5.9.0"
|
|
75
|
+
}
|
|
76
|
+
}
|