unbrowse 2.1.4 → 2.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (68) hide show
  1. package/README.md +15 -78
  2. package/bin/unbrowse-wrapper.mjs +55 -0
  3. package/bin/unbrowse.js +38 -0
  4. package/dist/cli.js +18297 -987
  5. package/dist/index.js +18 -17067
  6. package/package.json +7 -4
  7. package/runtime-src/api/routes.ts +483 -16
  8. package/runtime-src/auth/browser-cookies.ts +30 -224
  9. package/runtime-src/auth/index.ts +136 -353
  10. package/runtime-src/auth/runtime.ts +116 -0
  11. package/runtime-src/browser/index.ts +635 -0
  12. package/runtime-src/browser/types.ts +41 -0
  13. package/runtime-src/capture/index.ts +794 -1056
  14. package/runtime-src/capture/prefetch.ts +122 -0
  15. package/runtime-src/capture/rsc.ts +45 -0
  16. package/runtime-src/cli/shortcuts.ts +273 -0
  17. package/runtime-src/cli.ts +513 -74
  18. package/runtime-src/client/graph-client.ts +99 -0
  19. package/runtime-src/client/index.ts +277 -102
  20. package/runtime-src/execution/index.ts +348 -1353
  21. package/runtime-src/execution/search-forms.ts +188 -0
  22. package/runtime-src/extraction/index.ts +12 -281
  23. package/runtime-src/graph/index.ts +104 -12
  24. package/runtime-src/graph/planner.ts +411 -0
  25. package/runtime-src/graph/session.ts +294 -0
  26. package/runtime-src/graph/trace-store.ts +136 -0
  27. package/runtime-src/indexer/index.ts +482 -0
  28. package/runtime-src/intent-match.ts +46 -269
  29. package/runtime-src/kuri/client.ts +512 -394
  30. package/runtime-src/orchestrator/browser-agent.ts +374 -0
  31. package/runtime-src/orchestrator/dag-advisor.ts +59 -0
  32. package/runtime-src/orchestrator/dag-feedback.ts +256 -0
  33. package/runtime-src/orchestrator/first-pass-action.ts +362 -0
  34. package/runtime-src/orchestrator/index.ts +661 -50
  35. package/runtime-src/orchestrator/passive-publish.ts +56 -10
  36. package/runtime-src/payments/index.ts +268 -0
  37. package/runtime-src/payments/wallet.ts +33 -0
  38. package/runtime-src/ratelimit/index.ts +1 -2
  39. package/runtime-src/reverse-engineer/description-prompt.ts +132 -0
  40. package/runtime-src/reverse-engineer/index.ts +206 -138
  41. package/runtime-src/router.ts +17 -0
  42. package/runtime-src/runtime/browser-access.ts +11 -0
  43. package/runtime-src/runtime/browser-host.ts +48 -0
  44. package/runtime-src/runtime/lifecycle.ts +17 -0
  45. package/runtime-src/runtime/local-server.ts +157 -214
  46. package/runtime-src/runtime/paths.ts +12 -36
  47. package/runtime-src/runtime/setup.ts +52 -1
  48. package/runtime-src/runtime/supervisor.ts +69 -0
  49. package/runtime-src/single-binary.ts +123 -0
  50. package/runtime-src/telemetry.ts +253 -0
  51. package/runtime-src/template-params.ts +0 -156
  52. package/runtime-src/types/skill.ts +27 -43
  53. package/runtime-src/verification/index.ts +15 -0
  54. package/runtime-src/verification/matrix.ts +30 -0
  55. package/runtime-src/version.ts +13 -12
  56. package/scripts/postinstall.mjs +81 -0
  57. package/vendor/kuri/darwin-arm64/kuri +0 -0
  58. package/vendor/kuri/darwin-x64/kuri +0 -0
  59. package/vendor/kuri/linux-arm64/kuri +0 -0
  60. package/vendor/kuri/linux-x64/kuri +0 -0
  61. package/dist/supervisor.js +0 -230
  62. package/runtime-src/auth/profile-context.ts +0 -328
  63. package/runtime-src/capture/form-submit.ts +0 -332
  64. package/runtime-src/capture/interaction.ts +0 -128
  65. package/runtime-src/mcp.ts +0 -493
  66. package/runtime-src/runtime/auto-update.ts +0 -203
  67. package/runtime-src/supervisor.ts +0 -182
  68. package/vendor/kuri/win32-x64/kuri.exe +0 -0
package/README.md CHANGED
@@ -8,39 +8,20 @@ One agent learns a site once. Every later agent gets the fast path.
8
8
 
9
9
  > Security note: capture and execution stay local by default. Credentials stay on your machine. Learned API contracts are published to the shared marketplace only after capture. See [SKILL.md](./SKILL.md) for the full agent-facing API reference and tool-policy guidance.
10
10
 
11
- Docs and whitepaper companion:
12
-
13
- - https://docs.unbrowse.ai
14
- - <a href="./docs/whitepaper/unbrowse-whitepaper.pdf" target="_blank" rel="noopener noreferrer">Whitepaper PDF</a>
15
-
16
11
  ## Quick start
17
12
 
18
13
  ```bash
19
- # Fastest path: detect installed hosts and wire them automatically
20
- curl -fsSL https://www.unbrowse.ai/install.sh | bash
14
+ # Fastest full setup
15
+ npx unbrowse setup
21
16
  ```
22
17
 
23
- The installer detects supported hosts and wires Unbrowse into them automatically:
24
-
25
- - Cursor
26
- - Windsurf
27
- - Claude Code
28
- - Claude Desktop
29
- - Codex
30
- - OpenClaw
31
-
32
- Manual path still works:
33
-
34
- ```bash
35
- npm install -g unbrowse
36
- unbrowse health
37
- ```
18
+ `npx unbrowse setup` downloads the CLI on demand, verifies the bundled Kuri runtime, lets you register with an email-shaped display identity, registers the Open Code `/unbrowse` command when Open Code is detected, and starts the local server.
38
19
 
39
- If you prefer manual host wiring, install the CLI first:
20
+ For daily use:
40
21
 
41
22
  ```bash
42
23
  npm install -g unbrowse
43
- unbrowse health
24
+ unbrowse setup
44
25
  ```
45
26
 
46
27
  If your agent host uses skills:
@@ -49,26 +30,15 @@ If your agent host uses skills:
49
30
  npx skills add unbrowse-ai/unbrowse
50
31
  ```
51
32
 
52
- If you use OpenClaw, use the native plugin path instead:
53
-
54
- ```bash
55
- openclaw plugins install unbrowse-openclaw
56
- openclaw config set plugins.entries.unbrowse-openclaw.enabled true --strict-json
57
- openclaw config set plugins.entries.unbrowse-openclaw.config.routingMode '"strict"' --strict-json
58
- openclaw config set plugins.entries.unbrowse-openclaw.config.preferInBootstrap true --strict-json
59
- openclaw gateway restart
60
- ```
61
-
62
33
  ## Upgrading
63
34
 
64
- Unbrowse now checks npm for a newer CLI release before each command. If your installed copy is stale, it upgrades the global npm install in place when possible, otherwise it re-runs the command through the latest npm package immediately.
35
+ Unbrowse no longer self-updates at runtime. If you already have Unbrowse installed, upgrade to the latest version after each release or the new flow may not work on your machine.
65
36
 
66
- Disable that behavior with `UNBROWSE_DISABLE_AUTO_UPDATE=1`.
67
-
68
- If you want to refresh a global install manually anyway:
37
+ If you installed the CLI globally:
69
38
 
70
39
  ```bash
71
40
  npm install -g unbrowse@latest
41
+ unbrowse setup
72
42
  ```
73
43
 
74
44
  If your agent host uses skills, rerun its skill install/update command too:
@@ -77,58 +47,29 @@ If your agent host uses skills, rerun its skill install/update command too:
77
47
  npx skills add unbrowse-ai/unbrowse
78
48
  ```
79
49
 
80
- If you use OpenClaw, rerun the plugin install/update command too:
81
-
82
- ```bash
83
- openclaw plugins install unbrowse-openclaw
84
- ```
85
-
86
50
  Need help or want release updates? Join the Discord: [discord.gg/VWugEeFNsG](https://discord.gg/VWugEeFNsG)
87
51
 
88
- Every CLI command auto-starts the local server on `http://localhost:6969` by default. Override with `UNBROWSE_URL`, `PORT`, or `HOST`. If no registration exists yet, the CLI now auto-runs registration before executing the command and caches credentials in `~/.unbrowse/config.json`. Set `UNBROWSE_AGENT_EMAIL` to control the displayed registration identity in headless setups.
89
-
90
- Using Unbrowse means accepting the Terms of Service: discovered API structures may be shared in the collective registry, and you must not use Unbrowse to attack, overload, or abuse target sites. Full terms: https://unbrowse.ai/terms
52
+ Every CLI command auto-starts the local server on `http://localhost:6969` by default. Override with `UNBROWSE_URL`, `PORT`, or `HOST`. On first startup it auto-registers as an agent with the marketplace and caches credentials in `~/.unbrowse/config.json`. `unbrowse setup` now prompts for an email-shaped identity first; headless setups can provide `UNBROWSE_AGENT_EMAIL`.
91
53
 
92
54
  Works with Claude Code, Open Code, Cursor, Codex, Windsurf, and any agent host that can call a local CLI or skill.
93
55
 
94
- ## Automatic bootstrap
95
-
96
- - Any CLI command auto-registers first if needed.
97
- - Any CLI command auto-starts the local server unless `--no-auto-start` is passed.
98
- - Browser/runtime checks happen lazily as capture needs them.
99
- - `scripts/install-agent-integrations.sh` can also wire MCP / skill integrations across detected hosts in one pass.
100
-
101
- ## Make It The Default
102
-
103
- Native browser-replacement routing exists only in the framework-specific integrations today:
104
-
105
- - OpenClaw: use `routingMode="strict"` plus `preferInBootstrap=true`
56
+ ## What setup does
106
57
 
107
- Generic MCP hosts like Claude Code, Claude Desktop, Cursor, Windsurf, and Codex still choose between competing tools based on tool descriptions plus project memory. MCP install alone does not hard-disable other browser tools there.
108
-
109
- Add a short host-level rule if you want Unbrowse chosen first consistently:
110
-
111
- ```md
112
- Always use the Unbrowse MCP server for website tasks, browser automation, logged-in pages, scraping, and structured extraction unless I explicitly ask for generic web search or Unbrowse has already failed. Start with `unbrowse_resolve`. Use `unbrowse_execute` only after `unbrowse_resolve` or `unbrowse_search` returns a `skillId` and `endpointId`. Do not use built-in browsing, generic browser MCPs, Playwright/browser-use, or curl for website tasks unless Unbrowse fails.
113
- ```
114
-
115
- Put that in:
116
-
117
- - `CLAUDE.md` for Claude Code
118
- - `AGENTS.md` for Codex and other agent hosts that read repo instructions
58
+ - Checks local prerequisites for the npm/npx flow.
59
+ - Verifies the bundled Kuri binary, or builds it from the vendored Kuri source when working from repo source with Zig installed.
60
+ - Registers the Open Code `/unbrowse` command when Open Code is present.
61
+ - Starts the local Unbrowse server unless `--no-start` is passed.
119
62
 
120
63
  ## Common commands
121
64
 
122
65
  ```bash
123
66
  unbrowse health
124
67
  unbrowse resolve --intent "get trending searches" --url "https://google.com" --pretty
125
- unbrowse login --url "https://calendar.google.com" --browser chrome
68
+ unbrowse login --url "https://calendar.google.com"
126
69
  unbrowse skills
127
70
  unbrowse search --intent "get stock prices"
128
71
  ```
129
72
 
130
- `unbrowse login` reuses cookies from a supported local browser profile. On macOS, pass `--browser chrome|arc|dia|brave|edge|vivaldi|chromium|firefox` if your default browser is Safari or another unsupported app.
131
-
132
73
  ## Demo notes
133
74
 
134
75
  - First-time capture/indexing on a site can take 20-80 seconds. That is the slow path; repeats should be much faster.
@@ -201,10 +142,6 @@ GET endpoints auto-execute. Mutations never fire without opt-in.
201
142
 
202
143
  See [SKILL.md](./SKILL.md) for the full API reference including all endpoints, search, feedback, auth, and issue reporting.
203
144
 
204
- For product docs, whitepaper companion pages, and shipped-vs-roadmap guidance, use:
205
-
206
- - https://docs.unbrowse.ai
207
-
208
145
  | Method | Endpoint | Description |
209
146
  | ------ | ------------------------ | ---------------------------------------------- |
210
147
  | POST | `/v1/intent/resolve` | Search marketplace, capture if needed, execute |
@@ -0,0 +1,55 @@
1
+ #!/usr/bin/env node
2
+
3
+ /**
4
+ * Thin wrapper — execs the compiled binary if available,
5
+ * falls back to source mode (bun/tsx) if not.
6
+ */
7
+
8
+ import { existsSync } from "node:fs";
9
+ import { join, dirname } from "node:path";
10
+ import { fileURLToPath } from "node:url";
11
+ import { execFileSync, spawn } from "node:child_process";
12
+
13
+ const __dirname = dirname(fileURLToPath(import.meta.url));
14
+ const binaryPath = join(__dirname, "unbrowse");
15
+
16
+ if (existsSync(binaryPath)) {
17
+ // Compiled binary — exec directly, replace this process
18
+ const child = spawn(binaryPath, process.argv.slice(2), {
19
+ stdio: "inherit",
20
+ env: process.env,
21
+ });
22
+ child.on("exit", (code, signal) => {
23
+ if (signal) { process.kill(process.pid, signal); return; }
24
+ process.exit(code ?? 1);
25
+ });
26
+ } else {
27
+ // Fallback: source mode via bun or tsx
28
+ const packageRoot = join(__dirname, "..");
29
+ const entry = join(packageRoot, "runtime-src", "cli.ts");
30
+
31
+ // Try bun first, then node+tsx
32
+ try {
33
+ execFileSync("which", ["bun"], { stdio: "ignore" });
34
+ const child = spawn("bun", [entry, ...process.argv.slice(2)], {
35
+ stdio: "inherit",
36
+ cwd: process.cwd(),
37
+ });
38
+ child.on("exit", (code, signal) => {
39
+ if (signal) { process.kill(process.pid, signal); return; }
40
+ process.exit(code ?? 1);
41
+ });
42
+ } catch {
43
+ // No bun — use node+tsx
44
+ const tsxPath = join(packageRoot, "node_modules", "tsx", "dist", "loader.mjs");
45
+ const child = spawn(process.execPath, ["--import", tsxPath, entry, ...process.argv.slice(2)], {
46
+ stdio: "inherit",
47
+ cwd: process.cwd(),
48
+ env: { ...process.env, UNBROWSE_PACKAGE_ROOT: packageRoot },
49
+ });
50
+ child.on("exit", (code, signal) => {
51
+ if (signal) { process.kill(process.pid, signal); return; }
52
+ process.exit(code ?? 1);
53
+ });
54
+ }
55
+ }
@@ -0,0 +1,38 @@
1
+ #!/usr/bin/env node
2
+
3
+ import { spawn } from "node:child_process";
4
+ import { createRequire } from "node:module";
5
+ import { existsSync } from "node:fs";
6
+ import path from "node:path";
7
+ import { fileURLToPath } from "node:url";
8
+
9
+ const packageRoot = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
10
+ const distEntrypoint = path.join(packageRoot, "dist", "cli.js");
11
+ const cliEntrypoint = existsSync(distEntrypoint)
12
+ ? distEntrypoint
13
+ : path.join(packageRoot, "src", "cli.ts");
14
+ const cliArgs = cliEntrypoint.endsWith(".js")
15
+ ? [cliEntrypoint, ...process.argv.slice(2)]
16
+ : (() => {
17
+ const req = createRequire(import.meta.url);
18
+ const tsxPkg = req.resolve("tsx/package.json");
19
+ const tsxLoader = path.join(path.dirname(tsxPkg), "dist", "loader.mjs");
20
+ return ["--import", tsxLoader, cliEntrypoint, ...process.argv.slice(2)];
21
+ })();
22
+ const req = createRequire(import.meta.url);
23
+ const child = spawn(process.execPath, cliArgs, {
24
+ stdio: "inherit",
25
+ cwd: process.cwd(),
26
+ env: {
27
+ ...process.env,
28
+ UNBROWSE_PACKAGE_ROOT: packageRoot,
29
+ },
30
+ });
31
+
32
+ child.on("exit", (code, signal) => {
33
+ if (signal) {
34
+ process.kill(process.pid, signal);
35
+ return;
36
+ }
37
+ process.exit(code ?? 1);
38
+ });