pi-browser-harness 0.1.0 → 0.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +81 -0
- package/README.md +134 -112
- package/package.json +4 -1
- package/skills/pi-browser-harness/SKILL.md +17 -4
- package/src/cdp/discovery.ts +95 -0
- package/src/cdp/errors.ts +19 -0
- package/src/cdp/session.ts +139 -0
- package/src/cdp/transport.ts +221 -0
- package/src/cdp/types.ts +72 -0
- package/src/client.ts +232 -0
- package/src/domains/click.ts +73 -0
- package/src/domains/dialog.ts +31 -0
- package/src/domains/drag.ts +66 -0
- package/src/domains/files.ts +194 -0
- package/src/domains/history.ts +76 -0
- package/src/domains/js.ts +197 -0
- package/src/domains/keyboard.ts +127 -0
- package/src/domains/navigate.ts +146 -0
- package/src/domains/network.ts +80 -0
- package/src/domains/page.ts +88 -0
- package/src/domains/screenshot.ts +114 -0
- package/src/domains/scroll.ts +53 -0
- package/src/domains/tabs.ts +121 -0
- package/src/domains/viewport.ts +40 -0
- package/src/index.ts +65 -142
- package/src/prompt.ts +16 -3
- package/src/registry.ts +52 -0
- package/src/schemas/common.ts +11 -0
- package/src/setup.ts +20 -21
- package/src/state.ts +22 -54
- package/src/util/js-template.ts +27 -0
- package/src/util/keycodes.ts +10 -0
- package/src/util/paths.ts +21 -0
- package/src/util/result.ts +15 -0
- package/src/util/sharp-shim.ts +49 -0
- package/src/util/time.ts +29 -0
- package/src/util/tool.ts +169 -0
- package/src/util/truncate.ts +35 -0
- package/tsconfig.json +10 -1
- package/src/daemon.ts +0 -767
- package/src/protocol.ts +0 -77
- package/src/renderers.ts +0 -21
- package/src/tools.ts +0 -1358
package/CHANGELOG.md
CHANGED
|
@@ -2,6 +2,87 @@
|
|
|
2
2
|
|
|
3
3
|
All notable changes to pi-browser-harness will be documented in this file.
|
|
4
4
|
|
|
5
|
+
## 0.3.1 — 2026-05-02
|
|
6
|
+
|
|
7
|
+
### Bug fixes
|
|
8
|
+
|
|
9
|
+
- **`browser_list_tabs`** now shows full 32-character targetIds instead of truncated `BE9DD1DC…` prefixes. The `browser_list_tabs` → `browser_switch_tab` round-trip is repaired.
|
|
10
|
+
- **`browser_switch_tab`** now supports prefix matching: pass a unique hex prefix (≥8 chars) and it resolves to the full targetId automatically. Ambiguous prefixes return a clear error listing all matching tabs.
|
|
11
|
+
- **`browser_download`** auto-creates the download directory with `mkdir -p` if it doesn't exist. Previously it required a pre-existing writable directory.
|
|
12
|
+
- **`ensureAlive()`** now probes the page session with `Runtime.evaluate("1")` after the transport health check. If the page target has crashed (e.g. localhost server died), it reattaches automatically instead of returning a cryptic `session_not_found` error on the next tool call.
|
|
13
|
+
|
|
14
|
+
### Docs & metadata
|
|
15
|
+
|
|
16
|
+
- **`browser_dispatch_key`** prompt guidelines now explicitly note it dispatches a synthetic DOM `KeyboardEvent` and does NOT type text. Point users to `browser_type` / `browser_press_key` for actual text input.
|
|
17
|
+
- **`browser_navigate`** prompt guidelines now warn that Google and strict-anti-bot sites may reject CDP navigation. `browser_http_get` is the recommended workaround.
|
|
18
|
+
- **`browser_get_network_log`** prompt snippet updated with explicit workaround (`browser_execute_js` with `PerformanceObserver`).
|
|
19
|
+
- **`sharp`** added to `optionalDependencies` so `npm install` attempts it (enables `browser_screenshot` `maxDim` auto-resize).
|
|
20
|
+
- **`SKILL.md`** script bindings section updated to document the actual daemon API: `daemon.evaluateJs()`, `daemon.pageInfo()`, `daemon.listTabs()`, `daemon.session().call()`. Example script updated to match.
|
|
21
|
+
- **`SKILL.md`** troubleshooting section now covers `sharp`/`maxDim` and Google anti-bot navigation.
|
|
22
|
+
|
|
23
|
+
## 0.3.0 — 2026-05-02
|
|
24
|
+
|
|
25
|
+
### Internal rewrite
|
|
26
|
+
|
|
27
|
+
- Per-domain module split: every tool now lives in its own `src/domains/<name>.ts` file. The 1140-line `daemon.ts` and 2277-line `tools.ts` are gone; the three largest files are now `src/cdp/transport.ts` (~220 LOC), `src/client.ts` (~220 LOC), and `src/domains/js.ts` (~200 LOC).
|
|
28
|
+
- New transport/session/client split: `BrowserDaemon` class replaced with `createBrowserClient()` factory composing a `CdpTransport` (factory) and `CdpSession` (factory).
|
|
29
|
+
- All tool handlers now return `Result<T, E>`; one `defineBrowserTool` helper converts to pi's `ToolDefinition` and supplies a uniform `details` shape: `{ ok: true, ... }` on success, `{ ok: false, kind, message, ... }` on error.
|
|
30
|
+
- Strict TypeScript flags enabled (`noUncheckedIndexedAccess`, `exactOptionalPropertyTypes`, `noPropertyAccessFromIndexSignature`, `noUnusedLocals`, `noUnusedParameters`). Zero `any` in the codebase. All boundary `as` casts documented.
|
|
31
|
+
|
|
32
|
+
### Bug fixes (predictability)
|
|
33
|
+
|
|
34
|
+
- `browser_navigate` no longer silently creates a new tab when navigation fails; outcome is reported explicitly via `details.outcome.kind` (`"in_place"` | `"new_tab_created"`).
|
|
35
|
+
- Page-info cache invalidation is now automatic: the session subscribes to CDP `Page.frameNavigated` / `Page.loadEventFired` events. The 5 manual `invalidatePageInfoCache()` calls are gone.
|
|
36
|
+
- Dialog read no longer mutates state on `Page.javascriptDialogClosed` — the dialog persists in the buffer until `takeDialog()` is called, fixing a fast-dismiss race that dropped dialogs.
|
|
37
|
+
- `browser_dispatch_key` now returns `details.matched` (count of elements the synthetic event was dispatched to). Zero matches is an `invalid_state` error instead of a false success.
|
|
38
|
+
- `browser_http_get` timeout now covers the response body read (legacy aborted headers but `await response.text()` could hang indefinitely).
|
|
39
|
+
- `sharp` failures are distinguished from "sharp not installed" — actual errors no longer masked by the install hint.
|
|
40
|
+
- Screenshot paths use `randomUUID()` per-namespace — no more concurrent-write collisions from `Date.now() + global counter`.
|
|
41
|
+
- Reconnect is lazy (via `ensureAlive`) — no background reconnect, no stacked Chrome consent popups.
|
|
42
|
+
- WebSocket events are routed through an `AsyncIterable` bound to each connection — stale events from a previous connection can no longer leak across reconnects.
|
|
43
|
+
- `browser_wait_for_load` returns a typed `timeout` error if the deadline elapses (legacy returned a soft string).
|
|
44
|
+
|
|
45
|
+
### Security fixes
|
|
46
|
+
|
|
47
|
+
- All JS evaluation source is built via `safeJs\`...\`` (always JSON.stringify-safe). The previous `replace(/'/g, "\\'")` selector escaping (broken for backslashes, newlines, unicode quotes, `</script>`) is gone.
|
|
48
|
+
- `browser_run_script` now requires:
|
|
49
|
+
- script path inside `tmpdir()`, `cwd()`, or `BH_SCRIPT_DIR` (other paths rejected with `invalid_state`)
|
|
50
|
+
- a mandatory timeout (default 60s, max 600s, enforced via `Promise.race`)
|
|
51
|
+
- the AbortSignal is honored even if the script ignores its `signal` parameter
|
|
52
|
+
- source size ≤ 1 MB
|
|
53
|
+
- return shape validated structurally (each content item must be `{ type: "text", text: string }`)
|
|
54
|
+
- `browser_download` validates the directory exists and is writable before calling CDP (Chrome was silently downloading to nowhere if the dir was bogus).
|
|
55
|
+
- `browser_upload_file` verifies the file exists and is readable before any CDP call (prevents half-set state on the input).
|
|
56
|
+
- `pdfPath()` / `screenshotPath()` validate the namespace against a strict regex so a hostile namespace cannot escape `tmpdir()`.
|
|
57
|
+
|
|
58
|
+
### Parameter renames (saved scripts must be updated)
|
|
59
|
+
|
|
60
|
+
- `browser_click`: `clicks` → `count`
|
|
61
|
+
- `browser_dispatch_key`: `event` → `eventType`
|
|
62
|
+
|
|
63
|
+
### Removed
|
|
64
|
+
|
|
65
|
+
- The unused `tabHistory`, `screenshotDir`, and `debugClicks` fields on persisted state.
|
|
66
|
+
- The dead `tool_result` hook in `index.ts` for tab-history tracking (`details.targetId` was never set by any tool).
|
|
67
|
+
- `src/protocol.ts`, `src/renderers.ts`, `src/daemon.ts`, `src/tools.ts` (replaced by per-domain files and `client.ts`).
|
|
68
|
+
|
|
69
|
+
### Known follow-ups
|
|
70
|
+
|
|
71
|
+
- `browser_get_network_log` returns a structured deprecation note. The new transport routes events through an `AsyncIterable` consumed by the session manager; a synchronous `recentEvents()` API is deferred. Use `browser_execute_js` with `PerformanceObserver` or `performance.getEntries()` as a workaround.
|
|
72
|
+
- `browser_run_script` script binding is named `daemon` for back-compat, but the underlying object is now a `BrowserClient`. Scripts using `daemon.cdp(method, params)` should switch to `daemon.session().call(method, params)`.
|
|
73
|
+
- No tests added in this rewrite; that's a separate workstream.
|
|
74
|
+
|
|
75
|
+
## [0.2.0] - 2026-05-02
|
|
76
|
+
|
|
77
|
+
### Changed
|
|
78
|
+
- **Performance: fast `ensureAlive()`** — skips CDP `Target.getTargets` health-check roundtrip on every tool call. Uses WebSocket state check + 30s TTL. 96% faster per-call setup.
|
|
79
|
+
- **Performance: event-based `waitForLoad()`** — replaces `readyState` polling (300ms interval) with CDP `Page.loadEventFired` / `frameStoppedLoading` event draining (50ms interval). Detects already-loaded pages in ~1ms (99.5% faster).
|
|
80
|
+
- **Performance: JPEG screenshot support** — `captureScreenshot()` accepts `format` (png/jpeg) and `quality` (1-100). JPEG q80 is 29-49% smaller than PNG for complex pages, speeding up CDP transfer and reducing LLM context cost.
|
|
81
|
+
- **Performance: page info caching** — `getPageInfo()` caches results for 1 second, eliminating redundant `evaluateJS` CDP roundtrips on back-to-back calls.
|
|
82
|
+
- **Performance: parallel domain enables** — `switchTab()` enables Page/DOM/Runtime/Network domains via `Promise.all` instead of sequential await.
|
|
83
|
+
- **Tools: `browser_screenshot`** now accepts `format` (png/jpeg) and `quality` parameters.
|
|
84
|
+
- **Tools: `browser_wait_for_load`** now uses `daemon.waitForLoad()` (event-based) instead of polling.
|
|
85
|
+
|
|
5
86
|
## [0.1.0] - 2026-05-02
|
|
6
87
|
|
|
7
88
|
### Added
|
package/README.md
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# pi-browser-harness
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+

|
|
4
|
+
|
|
5
|
+
Full browser control for pi agents. Navigate, click, type, screenshot, and extract data from your real Chrome — all driven through natural language via CDP.
|
|
4
6
|
|
|
5
7
|
---
|
|
6
8
|
|
|
@@ -10,173 +12,163 @@ Full browser control for pi agents. Navigate, click, type, screenshot, and extra
|
|
|
10
12
|
# 1. Install
|
|
11
13
|
pi install npm:pi-browser-harness
|
|
12
14
|
|
|
13
|
-
# Or install from GitHub:
|
|
14
|
-
# pi install git:github.com/amankumarsingh77/pi-browser-harness
|
|
15
|
-
|
|
16
15
|
# 2. Enable Chrome remote debugging
|
|
17
16
|
# Open chrome://inspect/#remote-debugging in Chrome,
|
|
18
|
-
# tick
|
|
19
|
-
#
|
|
17
|
+
# tick "Discover network targets", click Allow.
|
|
18
|
+
# Or launch Chrome with --remote-debugging-port=9222
|
|
20
19
|
|
|
21
|
-
# 3. Connect
|
|
20
|
+
# 3. Connect
|
|
22
21
|
/browser-setup
|
|
23
22
|
```
|
|
24
23
|
|
|
25
|
-
That's it. Try:
|
|
26
|
-
|
|
27
|
-
```
|
|
28
|
-
> navigate to https://github.com and take a screenshot
|
|
29
|
-
```
|
|
30
|
-
|
|
31
24
|
### Requirements
|
|
32
25
|
|
|
33
26
|
- pi (latest)
|
|
34
27
|
- Node.js ≥ 22
|
|
35
|
-
- Chrome / Chromium / Edge
|
|
36
|
-
|
|
37
|
-
> If `chrome://inspect/#remote-debugging` isn't available, you can launch Chrome with `--remote-debugging-port=9222` instead. The harness will find it either way.
|
|
28
|
+
- Chrome / Chromium / Edge
|
|
38
29
|
|
|
39
30
|
---
|
|
40
31
|
|
|
41
|
-
##
|
|
32
|
+
## Tools
|
|
42
33
|
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
|
46
|
-
|
|
47
|
-
| `
|
|
34
|
+
| Tool | Purpose |
|
|
35
|
+
|------|---------|
|
|
36
|
+
| `browser_screenshot` | Capture a PNG screenshot of the current page |
|
|
37
|
+
| `browser_click` | Click at viewport coordinates |
|
|
38
|
+
| `browser_type` | Type text into the focused element |
|
|
39
|
+
| `browser_press_key` | Press a key (Enter, Tab, Escape, arrows, etc.) with optional modifiers |
|
|
40
|
+
| `browser_scroll` | Scroll the page by delta pixels |
|
|
41
|
+
| `browser_navigate` | Navigate to a URL |
|
|
48
42
|
| `browser_new_tab` | Open a new tab, optionally navigate |
|
|
49
|
-
| `browser_open_urls` | Open
|
|
50
|
-
| `browser_go_back` / `browser_go_forward` | History navigation |
|
|
51
|
-
| `
|
|
52
|
-
| `browser_list_tabs` / `browser_current_tab` |
|
|
53
|
-
| `
|
|
54
|
-
| `
|
|
55
|
-
|
|
56
|
-
### Interaction
|
|
57
|
-
|
|
58
|
-
| Tool | What it does |
|
|
59
|
-
|------|--------------|
|
|
60
|
-
| `browser_click` | Click at viewport coordinates (works through iframes / shadow DOM) |
|
|
61
|
-
| `browser_type` | Type into the focused element |
|
|
62
|
-
| `browser_press_key` | Send a key with optional modifiers |
|
|
63
|
-
| `browser_scroll` | Scroll by delta pixels |
|
|
64
|
-
| `browser_handle_dialog` | Accept or dismiss `alert` / `confirm` / `prompt` |
|
|
65
|
-
|
|
66
|
-
### Visual & data
|
|
67
|
-
|
|
68
|
-
| Tool | What it does |
|
|
69
|
-
|------|--------------|
|
|
70
|
-
| `browser_screenshot` | Capture the current page as PNG |
|
|
71
|
-
| `browser_execute_js` | Run JS in the page and return the result |
|
|
72
|
-
| `browser_http_get` | Direct HTTP GET — bypasses the browser, much faster for APIs |
|
|
43
|
+
| `browser_open_urls` | Open multiple URLs in parallel tabs |
|
|
44
|
+
| `browser_go_back` / `browser_go_forward` / `browser_reload` | History navigation |
|
|
45
|
+
| `browser_page_info` | Get URL, title, viewport, scroll position, or dialog state |
|
|
46
|
+
| `browser_list_tabs` / `browser_current_tab` / `browser_switch_tab` | Tab discovery and switching |
|
|
47
|
+
| `browser_execute_js` | Run JavaScript in the page and return the result |
|
|
48
|
+
| `browser_http_get` | Direct HTTP GET outside the browser — much faster for APIs |
|
|
73
49
|
| `browser_wait` / `browser_wait_for_load` | Sleep, or wait for `readyState === 'complete'` |
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
| Tool | What it does |
|
|
78
|
-
|------|--------------|
|
|
79
|
-
| `browser_run_script` | Execute a temporary script file with daemon access |
|
|
50
|
+
| `browser_handle_dialog` | Accept or dismiss `alert` / `confirm` / `prompt` |
|
|
51
|
+
| `browser_run_script` | Execute a temporary script with daemon and Node.js access |
|
|
80
52
|
|
|
81
53
|
---
|
|
82
54
|
|
|
83
|
-
##
|
|
55
|
+
## Core Patterns
|
|
84
56
|
|
|
85
|
-
The
|
|
57
|
+
The golden rule: **screenshot → act → screenshot → verify.** Never assume — always look.
|
|
86
58
|
|
|
87
|
-
###
|
|
59
|
+
### Navigation
|
|
88
60
|
|
|
89
61
|
```
|
|
90
|
-
|
|
62
|
+
browser_new_tab({ url: "https://example.com" })
|
|
91
63
|
browser_wait_for_load()
|
|
92
64
|
browser_screenshot()
|
|
93
65
|
```
|
|
94
66
|
|
|
95
|
-
###
|
|
67
|
+
### Form filling
|
|
96
68
|
|
|
97
69
|
```
|
|
98
|
-
browser_screenshot()
|
|
99
|
-
browser_click(400, 200)
|
|
100
|
-
browser_type(
|
|
101
|
-
browser_press_key("Enter")
|
|
70
|
+
browser_screenshot() # see the form
|
|
71
|
+
browser_click({ x: 400, y: 200 }) # click the search input
|
|
72
|
+
browser_type({ text: "query" }) # type
|
|
73
|
+
browser_press_key({ key: "Enter" }) # submit
|
|
102
74
|
browser_wait_for_load()
|
|
103
|
-
browser_screenshot()
|
|
75
|
+
browser_screenshot() # verify
|
|
104
76
|
```
|
|
105
77
|
|
|
106
|
-
###
|
|
78
|
+
### Data extraction
|
|
107
79
|
|
|
108
|
-
```
|
|
109
|
-
|
|
110
|
-
browser_http_get("https://api.github.com/repos/browser-use/browser-harness")
|
|
80
|
+
```
|
|
81
|
+
browser_execute_js({ expression: "document.querySelector('.price').innerText" })
|
|
111
82
|
|
|
112
|
-
|
|
113
|
-
browser_execute_js("document.querySelector('.price').innerText")
|
|
83
|
+
browser_http_get({ url: "https://api.github.com/repos/amankumarsingh77/pi-browser-harness" })
|
|
114
84
|
|
|
115
85
|
// Structured arrays
|
|
116
|
-
browser_execute_js(`
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
link: el.querySelector('a')?.href,
|
|
121
|
-
}))
|
|
122
|
-
)
|
|
123
|
-
`)
|
|
86
|
+
browser_execute_js({ expression: `Array.from(document.querySelectorAll('.result')).map(el => ({
|
|
87
|
+
title: el.querySelector('h3')?.textContent,
|
|
88
|
+
link: el.querySelector('a')?.href,
|
|
89
|
+
}))` })
|
|
124
90
|
```
|
|
125
91
|
|
|
126
|
-
###
|
|
92
|
+
### Research workflow
|
|
127
93
|
|
|
128
94
|
```
|
|
129
|
-
browser_navigate("https://google.com/search?q
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
"https://developer.chrome.com/docs/devtools",
|
|
133
|
-
]})
|
|
95
|
+
browser_navigate({ url: "https://google.com/search?q=..." })
|
|
96
|
+
// read search results
|
|
97
|
+
browser_open_urls({ urls: ["url1", "url2", "url3"] })
|
|
134
98
|
browser_list_tabs()
|
|
135
99
|
browser_switch_tab({ targetId: "..." })
|
|
100
|
+
browser_screenshot()
|
|
101
|
+
browser_execute_js({ expression: "document.querySelector('.content').innerText" })
|
|
136
102
|
```
|
|
137
103
|
|
|
138
|
-
###
|
|
139
|
-
|
|
140
|
-
`browser_wait_for_load` only checks `readyState`. For React/Vue/etc., poll for the actual element you want:
|
|
104
|
+
### Scrolling
|
|
141
105
|
|
|
142
|
-
```
|
|
143
|
-
|
|
106
|
+
```
|
|
107
|
+
browser_screenshot()
|
|
108
|
+
browser_scroll({ deltaY: -500 })
|
|
109
|
+
browser_screenshot()
|
|
144
110
|
```
|
|
145
111
|
|
|
146
112
|
### Keyboard modifiers
|
|
147
113
|
|
|
148
|
-
|
|
114
|
+
| Key | Bit |
|
|
115
|
+
|-----|-----|
|
|
116
|
+
| Alt | 1 |
|
|
117
|
+
| Ctrl | 2 |
|
|
118
|
+
| Meta / Cmd | 4 |
|
|
119
|
+
| Shift | 8 |
|
|
149
120
|
|
|
121
|
+
Combine with bitwise OR:
|
|
150
122
|
```
|
|
151
|
-
browser_press_key("c",
|
|
152
|
-
browser_press_key("v",
|
|
153
|
-
browser_press_key("T",
|
|
123
|
+
browser_press_key({ key: "c", modifiers: 2 }) // Ctrl+C
|
|
124
|
+
browser_press_key({ key: "v", modifiers: 4 }) // Cmd+V
|
|
125
|
+
browser_press_key({ key: "T", modifiers: 10 }) // Ctrl+Shift+T
|
|
154
126
|
```
|
|
155
127
|
|
|
156
|
-
###
|
|
128
|
+
### SPAs (React, Vue, etc.)
|
|
157
129
|
|
|
158
|
-
|
|
130
|
+
`browser_wait_for_load` only checks `readyState`. Poll for elements after:
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
browser_execute_js({ expression: "!!document.querySelector('.loaded-content')" })
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### Dialogs
|
|
137
|
+
|
|
138
|
+
JS dialogs (`alert` / `confirm` / `prompt`) freeze the page. Check `browser_page_info` first — if it reports a dialog, handle it before anything else:
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
browser_handle_dialog({ action: "accept" }) // confirm
|
|
142
|
+
browser_handle_dialog({ action: "dismiss" }) // cancel
|
|
143
|
+
browser_handle_dialog({ action: "accept", promptText: "hello" }) // prompt
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
---
|
|
147
|
+
|
|
148
|
+
## Temporary Scripts
|
|
149
|
+
|
|
150
|
+
When built-in tools aren't enough, write a script to disk and run it:
|
|
159
151
|
|
|
160
152
|
```js
|
|
161
|
-
// 1. Write the script
|
|
162
|
-
write("/tmp/scrape-
|
|
163
|
-
const
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
)
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
details: { items: JSON.parse(data) },
|
|
172
|
-
};
|
|
153
|
+
// 1. Write the script
|
|
154
|
+
write("/tmp/scrape-pages.js", `
|
|
155
|
+
const results = [];
|
|
156
|
+
for (const url of params.urls) {
|
|
157
|
+
await daemon.cdp("Page.navigate", { url });
|
|
158
|
+
await new Promise(r => setTimeout(r, 2000));
|
|
159
|
+
const title = await daemon.evaluateJS("document.title");
|
|
160
|
+
results.push({ url, title });
|
|
161
|
+
}
|
|
162
|
+
return { content: [{ type: "text", text: JSON.stringify(results, null, 2) }] };
|
|
173
163
|
`)
|
|
174
164
|
|
|
175
165
|
// 2. Execute it
|
|
176
|
-
browser_run_script("/tmp/scrape-
|
|
166
|
+
browser_run_script({ path: "/tmp/scrape-pages.js", params: { urls: [...] } })
|
|
177
167
|
```
|
|
178
168
|
|
|
179
|
-
|
|
169
|
+
**Script bindings:** `params`, `daemon`, `require`, `signal`, `onUpdate`, `ctx`, `console`, `fetch`, `JSON`, `Buffer`, `setTimeout`, `clearTimeout`.
|
|
170
|
+
|
|
171
|
+
Scripts are written to disk — auditable and re-runnable.
|
|
180
172
|
|
|
181
173
|
---
|
|
182
174
|
|
|
@@ -190,20 +182,50 @@ Scripts receive `params`, `daemon`, `require`, `signal`, `onUpdate`, `ctx`, plus
|
|
|
190
182
|
|
|
191
183
|
---
|
|
192
184
|
|
|
185
|
+
## What NOT to do
|
|
186
|
+
|
|
187
|
+
- Don't launch your own browser — you're connected to the user's real Chrome
|
|
188
|
+
- Don't type credentials — if you hit an auth wall, stop and ask
|
|
189
|
+
- Don't assume a page action worked — screenshot to verify
|
|
190
|
+
- Don't write pixel coordinates into documentation — describe selectors and text instead
|
|
191
|
+
- Don't ignore dialogs — check `browser_page_info` first
|
|
192
|
+
|
|
193
|
+
---
|
|
194
|
+
|
|
195
|
+
## Architecture
|
|
196
|
+
|
|
197
|
+
```
|
|
198
|
+
pi agent → pi-browser-harness (TypeScript)
|
|
199
|
+
│ CDP WebSocket
|
|
200
|
+
▼
|
|
201
|
+
Chrome
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
Temporary scripts run inside the harness process with full daemon and Node.js access.
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
193
208
|
## Troubleshooting
|
|
194
209
|
|
|
195
210
|
| Symptom | Fix |
|
|
196
211
|
|---------|-----|
|
|
197
|
-
| `DevToolsActivePort not found` | Open `chrome://inspect/#remote-debugging`, tick the checkbox, click Allow
|
|
198
|
-
| Connection fails after Chrome restart | Run `/browser-reload-daemon
|
|
199
|
-
| Page seems loaded but content is missing | SPA —
|
|
200
|
-
| A JS dialog is blocking actions | `browser_page_info` will report
|
|
212
|
+
| `DevToolsActivePort not found` | Open `chrome://inspect/#remote-debugging`, tick the checkbox, click Allow |
|
|
213
|
+
| Connection fails after Chrome restart | Run `/browser-reload-daemon` |
|
|
214
|
+
| Page seems loaded but content is missing | SPA — poll with `browser_execute_js` for the element you need |
|
|
215
|
+
| A JS dialog is blocking actions | `browser_page_info` will report it — use `browser_handle_dialog` |
|
|
216
|
+
| Daemon not starting | Run `/browser-setup` to re-run guided setup |
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
## Contributing
|
|
221
|
+
|
|
222
|
+
See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, conventions, and PR process.
|
|
201
223
|
|
|
202
224
|
---
|
|
203
225
|
|
|
204
226
|
## Security
|
|
205
227
|
|
|
206
|
-
This extension drives your real Chrome. The agent can see open tabs, read page content, submit forms, and act inside authenticated sessions. `browser_run_script` evaluates JavaScript in the pi process with full `require` access — review any temporary scripts
|
|
228
|
+
This extension drives your real Chrome. The agent can see open tabs, read page content, submit forms, and act inside authenticated sessions. `browser_run_script` evaluates JavaScript in the pi process with full `require` access — review any temporary scripts before executing them.
|
|
207
229
|
|
|
208
230
|
---
|
|
209
231
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "pi-browser-harness",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.3.1",
|
|
4
4
|
"description": "Browser control extension for pi — navigate, click, type, screenshot, and extract data from real Chrome via CDP",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"pi-package",
|
|
@@ -62,6 +62,9 @@
|
|
|
62
62
|
"publishConfig": {
|
|
63
63
|
"access": "public"
|
|
64
64
|
},
|
|
65
|
+
"optionalDependencies": {
|
|
66
|
+
"sharp": "^0.33.0"
|
|
67
|
+
},
|
|
65
68
|
"dependencies": {
|
|
66
69
|
"ws": "^8.20.0"
|
|
67
70
|
}
|
|
@@ -130,7 +130,18 @@ The script is written to disk, so it's auditable and re-runnable.
|
|
|
130
130
|
|
|
131
131
|
**Script bindings:**
|
|
132
132
|
- `params` — the arguments passed to browser_run_script
|
|
133
|
-
- `daemon` — browser daemon
|
|
133
|
+
- `daemon` — browser daemon with these methods:
|
|
134
|
+
- `daemon.evaluateJs(expression)` — run JS in the current page
|
|
135
|
+
- `daemon.pageInfo()` — get current page info or dialog
|
|
136
|
+
- `daemon.listTabs()` — returns `{ success, data: [{ targetId, url, title }] }`
|
|
137
|
+
- `daemon.switchTab(targetId)` — switch to a tab by full targetId
|
|
138
|
+
- `daemon.newTab(url?)` — open and switch to a new tab
|
|
139
|
+
- `daemon.current()` — returns `{ targetId, url?, title? }`
|
|
140
|
+
- `daemon.session(targetId)` — returns a session object for raw CDP:
|
|
141
|
+
- `session.call(method, params?)` — CDP command on the page target
|
|
142
|
+
- `session.callOnTarget(method, params, sessionId)` — CDP on a specific session
|
|
143
|
+
- `session.callBrowser(method, params?)` — CDP on the browser target
|
|
144
|
+
- `session.takeDialog()` — get/dismiss pending dialog
|
|
134
145
|
- `require` — Node.js require() (use 'fs', 'path', 'crypto', etc.)
|
|
135
146
|
- `signal` — AbortSignal for cancellation
|
|
136
147
|
- `onUpdate` — progress callback: onUpdate({ content: [{ type: 'text', text: '...' }] })
|
|
@@ -140,9 +151,9 @@ The script is written to disk, so it's auditable and re-runnable.
|
|
|
140
151
|
**Example — a script that extracts structured data:**
|
|
141
152
|
```javascript
|
|
142
153
|
// /tmp/extract-products.js
|
|
143
|
-
const info = await daemon.
|
|
144
|
-
if ("dialog" in info) throw new Error("Dialog is blocking: " + info.dialog.message);
|
|
145
|
-
const data = await daemon.
|
|
154
|
+
const info = await daemon.pageInfo();
|
|
155
|
+
if (info && "dialog" in info) throw new Error("Dialog is blocking: " + info.dialog.message);
|
|
156
|
+
const data = await daemon.evaluateJs(`
|
|
146
157
|
JSON.stringify(Array.from(document.querySelectorAll('${params.rowSelector}')).map(el => ({
|
|
147
158
|
title: el.querySelector('${params.titleSelector}')?.textContent?.trim(),
|
|
148
159
|
price: el.querySelector('${params.priceSelector}')?.textContent?.trim(),
|
|
@@ -196,6 +207,8 @@ browser_execute_js("!!document.querySelector('.loaded-content')")
|
|
|
196
207
|
- **Stale session**: Run `/browser-reload-daemon` to restart the daemon
|
|
197
208
|
- **Dialog blocking**: Run browser_page_info to detect it, then browser_handle_dialog
|
|
198
209
|
- **Status check**: Run `/browser-status` to see daemon health and current page
|
|
210
|
+
- **Screenshot maxDim not working**: Install sharp (`npm install sharp`) for auto-resize support
|
|
211
|
+
- **Google navigation fails**: Google's anti-bot detection may reject CDP navigation; use `browser_http_get` to fetch search results instead
|
|
199
212
|
|
|
200
213
|
## Architecture
|
|
201
214
|
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
import { readFile } from "node:fs/promises";
|
|
2
|
+
import { homedir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
import { connect as netConnect } from "node:net";
|
|
5
|
+
import { type Result, err, ok } from "../util/result";
|
|
6
|
+
import { type CdpError, cdpError } from "./errors";
|
|
7
|
+
|
|
8
|
+
const PORT_PROBE_DEADLINE_MS = 30_000;
|
|
9
|
+
const PORT_PROBE_INTERVAL_MS = 1_000;
|
|
10
|
+
|
|
11
|
+
const profileDirs = (): ReadonlyArray<string> => {
|
|
12
|
+
const home = homedir();
|
|
13
|
+
return [
|
|
14
|
+
join(home, "Library/Application Support/Google/Chrome"),
|
|
15
|
+
join(home, "Library/Application Support/Microsoft Edge"),
|
|
16
|
+
join(home, "Library/Application Support/Microsoft Edge Beta"),
|
|
17
|
+
join(home, "Library/Application Support/Microsoft Edge Dev"),
|
|
18
|
+
join(home, "Library/Application Support/Microsoft Edge Canary"),
|
|
19
|
+
join(home, ".config/google-chrome"),
|
|
20
|
+
join(home, ".config/chromium"),
|
|
21
|
+
join(home, ".config/chromium-browser"),
|
|
22
|
+
join(home, ".config/microsoft-edge"),
|
|
23
|
+
join(home, ".config/microsoft-edge-beta"),
|
|
24
|
+
join(home, ".config/microsoft-edge-dev"),
|
|
25
|
+
join(home, ".var/app/org.chromium.Chromium/config/chromium"),
|
|
26
|
+
join(home, ".var/app/com.google.Chrome/config/google-chrome"),
|
|
27
|
+
join(home, ".var/app/com.brave.Browser/config/BraveSoftware/Brave-Browser"),
|
|
28
|
+
join(home, ".var/app/com.microsoft.Edge/config/microsoft-edge"),
|
|
29
|
+
join(home, "AppData/Local/Google/Chrome/User Data"),
|
|
30
|
+
join(home, "AppData/Local/Chromium/User Data"),
|
|
31
|
+
join(home, "AppData/Local/Microsoft/Edge/User Data"),
|
|
32
|
+
join(home, "AppData/Local/Microsoft/Edge Beta/User Data"),
|
|
33
|
+
join(home, "AppData/Local/Microsoft/Edge Dev/User Data"),
|
|
34
|
+
join(home, "AppData/Local/Microsoft/Edge SxS/User Data"),
|
|
35
|
+
];
|
|
36
|
+
};
|
|
37
|
+
|
|
38
|
+
const probePort = (port: number): Promise<Result<void, CdpError>> =>
|
|
39
|
+
new Promise((resolve) => {
|
|
40
|
+
const sock = netConnect({ host: "127.0.0.1", port });
|
|
41
|
+
let settled = false;
|
|
42
|
+
const finish = (r: Result<void, CdpError>): void => {
|
|
43
|
+
if (settled) return;
|
|
44
|
+
settled = true;
|
|
45
|
+
sock.setTimeout(0);
|
|
46
|
+
sock.destroy();
|
|
47
|
+
resolve(r);
|
|
48
|
+
};
|
|
49
|
+
sock.setTimeout(1000, () => finish(err(cdpError("discovery_failed", "probe timeout"))));
|
|
50
|
+
sock.once("error", (e) => finish(err(cdpError("discovery_failed", e.message))));
|
|
51
|
+
sock.once("connect", () => finish(ok(undefined)));
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
const waitForPort = async (port: number): Promise<Result<void, CdpError>> => {
|
|
55
|
+
const end = Date.now() + PORT_PROBE_DEADLINE_MS;
|
|
56
|
+
let lastMessage = "unknown";
|
|
57
|
+
while (Date.now() < end) {
|
|
58
|
+
const probe = await probePort(port);
|
|
59
|
+
if (probe.success) return probe;
|
|
60
|
+
lastMessage = probe.error.message;
|
|
61
|
+
await new Promise((r) => setTimeout(r, PORT_PROBE_INTERVAL_MS));
|
|
62
|
+
}
|
|
63
|
+
return err(cdpError(
|
|
64
|
+
"discovery_failed",
|
|
65
|
+
`Chrome's remote-debugging page is open, but DevTools is not live yet on 127.0.0.1:${port} — if Chrome opened a profile picker, choose your normal profile first, then tick the checkbox and click Allow if shown (last error: ${lastMessage})`,
|
|
66
|
+
));
|
|
67
|
+
};
|
|
68
|
+
|
|
69
|
+
export const discoverWsUrl = async (): Promise<Result<string, CdpError>> => {
|
|
70
|
+
const dirs = profileDirs();
|
|
71
|
+
for (const base of dirs) {
|
|
72
|
+
const portFile = join(base, "DevToolsActivePort");
|
|
73
|
+
let raw: string;
|
|
74
|
+
try {
|
|
75
|
+
raw = await readFile(portFile, "utf8");
|
|
76
|
+
} catch (e) {
|
|
77
|
+
// Node fs errors carry .code; see ErrnoException
|
|
78
|
+
const code = (e as NodeJS.ErrnoException).code;
|
|
79
|
+
if (code === "ENOENT" || code === undefined) continue;
|
|
80
|
+
return err(cdpError("discovery_failed", `failed to read ${portFile}: ${e instanceof Error ? e.message : String(e)}`));
|
|
81
|
+
}
|
|
82
|
+
const lines = raw.trim().split("\n");
|
|
83
|
+
if (lines.length < 2) continue;
|
|
84
|
+
const port = lines[0]?.trim();
|
|
85
|
+
const path = lines[1]?.trim();
|
|
86
|
+
if (!port || !path) continue;
|
|
87
|
+
const ready = await waitForPort(Number(port));
|
|
88
|
+
if (!ready.success) return ready;
|
|
89
|
+
return ok(`ws://127.0.0.1:${port}${path}`);
|
|
90
|
+
}
|
|
91
|
+
return err(cdpError(
|
|
92
|
+
"discovery_failed",
|
|
93
|
+
`DevToolsActivePort not found in ${dirs.join(", ")} — open chrome://inspect/#remote-debugging in your browser, tick the checkbox, click Allow, then retry. Or set BU_CDP_WS to a remote browser endpoint.`,
|
|
94
|
+
));
|
|
95
|
+
};
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
export type CdpErrorKind =
|
|
2
|
+
| "transport_closed"
|
|
3
|
+
| "timeout"
|
|
4
|
+
| "session_not_found"
|
|
5
|
+
| "remote_error"
|
|
6
|
+
| "discovery_failed"
|
|
7
|
+
| "invalid_response";
|
|
8
|
+
|
|
9
|
+
export type CdpError = {
|
|
10
|
+
readonly kind: CdpErrorKind;
|
|
11
|
+
readonly message: string;
|
|
12
|
+
readonly method?: string;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export const cdpError = (
|
|
16
|
+
kind: CdpErrorKind,
|
|
17
|
+
message: string,
|
|
18
|
+
method?: string,
|
|
19
|
+
): CdpError => ({ kind, message, ...(method !== undefined ? { method } : {}) });
|