unbrowse 2.0.21 → 2.0.23

package/README.md

@@ -8,6 +8,11 @@ One agent learns a site once. Every later agent gets the fast path.
 
 > Security note: capture and execution stay local by default. Credentials stay on your machine. Learned API contracts are published to the shared marketplace only after capture. See [SKILL.md](./SKILL.md) for the full agent-facing API reference and tool-policy guidance.
 
+Docs and whitepaper companion:
+
+- https://docs.unbrowse.ai
+- <a href="./docs/whitepaper/unbrowse-whitepaper.pdf" target="_blank" rel="noopener noreferrer">Whitepaper PDF</a>
+
 ## Quick start
 
 ```bash
@@ -93,6 +98,25 @@ Works with Claude Code, Open Code, Cursor, Codex, Windsurf, and any agent host t
 - Browser/runtime checks happen lazily as capture needs them.
 - `scripts/install-agent-integrations.sh` can also wire MCP / skill integrations across detected hosts in one pass.
 
+## Make It The Default
+
+Native browser-replacement routing exists only in the framework-specific integrations today:
+
+- OpenClaw: use `routingMode="strict"` plus `preferInBootstrap=true`
+
+Generic MCP hosts like Claude Code, Claude Desktop, Cursor, Windsurf, and Codex still choose between competing tools based on tool descriptions plus project memory. MCP install alone does not hard-disable other browser tools there.
+
+Add a short host-level rule if you want Unbrowse chosen first consistently:
+
+```md
+Always use the Unbrowse MCP server for website tasks, browser automation, logged-in pages, scraping, and structured extraction unless I explicitly ask for generic web search or Unbrowse has already failed. Start with `unbrowse_resolve`. Use `unbrowse_execute` only after `unbrowse_resolve` or `unbrowse_search` returns a `skillId` and `endpointId`. Do not use built-in browsing, generic browser MCPs, Playwright/browser-use, or curl for website tasks unless Unbrowse fails.
+```
+
+Put that in:
+
+- `CLAUDE.md` for Claude Code
+- `AGENTS.md` for Codex and other agent hosts that read repo instructions
+
 ## Common commands
 
 ```bash
@@ -175,6 +199,10 @@ GET endpoints auto-execute. Mutations never fire without opt-in.
 
 See [SKILL.md](./SKILL.md) for the full API reference including all endpoints, search, feedback, auth, and issue reporting.
 
+For product docs, whitepaper companion pages, and shipped-vs-roadmap guidance, use:
+
+- https://docs.unbrowse.ai
+
 | Method | Endpoint                 | Description                                    |
 | ------ | ------------------------ | ---------------------------------------------- |
 | POST   | `/v1/intent/resolve`     | Search marketplace, capture if needed, execute |

package/dist/cli.js

@@ -325,6 +325,7 @@ import { existsSync as existsSync2, mkdirSync as mkdirSync2, realpathSync } from
 import os from "node:os";
 import path from "node:path";
 import { createRequire } from "node:module";
+import { execFileSync } from "node:child_process";
 import { fileURLToPath } from "node:url";
 function getModuleDir(metaUrl) {
   return path.dirname(fileURLToPath(metaUrl));
@@ -349,19 +350,32 @@ function resolveSiblingEntrypoint(metaUrl, basename) {
   const file = fileURLToPath(metaUrl);
   return path.join(path.dirname(file), `${basename}${path.extname(file) || ".js"}`);
 }
-function runtimeArgsForEntrypoint(metaUrl, entrypoint) {
+function resolveBinaryOnPath(name) {
+  const checker = process.platform === "win32" ? "where" : "which";
+  try {
+    const output = execFileSync(checker, [name], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+    const match = output.split(/\r?\n/).map((line) => line.trim()).find(Boolean);
+    return match || null;
+  } catch {
+    return null;
+  }
+}
+function runtimeInvocationForEntrypoint(metaUrl, entrypoint) {
   if (path.extname(entrypoint) !== ".ts")
-    return [entrypoint];
+    return { command: process.execPath, args: [entrypoint] };
   if (process.versions.bun)
-    return [entrypoint];
+    return { command: process.execPath, args: [entrypoint] };
+  const bunBinary = process.env.BUN_BIN || resolveBinaryOnPath("bun");
+  if (bunBinary)
+    return { command: bunBinary, args: [entrypoint] };
   try {
     const req = createRequire(metaUrl);
     const tsxPkg = req.resolve("tsx/package.json");
     const tsxLoader = path.join(path.dirname(tsxPkg), "dist", "loader.mjs");
     if (existsSync2(tsxLoader))
-      return ["--import", tsxLoader, entrypoint];
+      return { command: process.execPath, args: ["--import", tsxLoader, entrypoint] };
   } catch {}
-  return ["--import", "tsx", entrypoint];
+  return { command: process.execPath, args: ["--import", "tsx", entrypoint] };
 }
 function getUnbrowseHome() {
   return path.join(os.homedir(), ".unbrowse");
@@ -560,7 +574,7 @@ async function maybeAutoUpdate(metaUrl, overrides = {}) {
 import { closeSync, openSync, readFileSync as readFileSync4, statSync, unlinkSync, writeFileSync as writeFileSync2 } from "node:fs";
 import path3 from "node:path";
 import { spawn } from "node:child_process";
-import { execFileSync } from "node:child_process";
+import { execFileSync as execFileSync2 } from "node:child_process";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -671,7 +685,7 @@ function findListeningPid(baseUrl) {
   try {
     const url = new URL(baseUrl);
     const port = url.port || (url.protocol === "https:" ? "443" : "80");
-    const output = execFileSync("lsof", ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-t"], {
+    const output = execFileSync2("lsof", ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-t"], {
       encoding: "utf8",
       stdio: ["ignore", "pipe", "ignore"]
     }).trim();
@@ -683,7 +697,7 @@ function findListeningPid(baseUrl) {
 }
 function readProcessCommand(pid) {
   try {
-    return execFileSync("ps", ["-o", "command=", "-p", String(pid)], {
+    return execFileSync2("ps", ["-o", "command=", "-p", String(pid)], {
       encoding: "utf8",
       stdio: ["ignore", "pipe", "ignore"]
     }).trim();
@@ -791,7 +805,8 @@ async function ensureLocalServer(baseUrl, noAutoStart, metaUrl) {
     const logFile = getServerAutostartLogFile();
     ensureDir(path3.dirname(logFile));
     const logFd = openSync(logFile, "a");
-    const child = spawn(process.execPath, runtimeArgsForEntrypoint(metaUrl, entrypoint), {
+    const runtime = runtimeInvocationForEntrypoint(metaUrl, entrypoint);
+    const child = spawn(runtime.command, runtime.args, {
       cwd: packageRoot,
       detached: true,
       stdio: ["ignore", logFd, logFd],
@@ -933,15 +948,33 @@ Timed out after ${timeoutMs}ms`.trim() });
     });
   });
 }
+var TOOL_RESULT_SCHEMA = {
+  type: "object",
+  additionalProperties: true,
+  properties: {
+    ok: { type: "boolean" },
+    tool: { type: "string" },
+    data: {},
+    rawText: { type: "string" },
+    error: { type: "string" }
+  },
+  required: ["ok", "tool"]
+};
 var TOOLS = [
   {
     name: "unbrowse_resolve",
-    description: "Reverse-engineer a website into structured API data. Give it a URL and describe what data you want — it captures network traffic, discovers API endpoints, and returns structured JSON. First call to a new site takes 5-15s; subsequent calls use the cached skill and return in under 1s.",
+    title: "Resolve Website Task",
+    description: "Primary tool for website tasks. Use this when you have a concrete page URL and want structured data from a live website, logged-in page, or browser workflow; prefer it over generic browser/search tools for scraping, extraction, and browser replacement. Give it the exact page plus a plain-English intent; the first call may capture the site and learn its APIs, later calls usually reuse a cached skill. Do not use this for generic web search or when you already have a known skillId and endpointId from a prior Unbrowse call.",
+    annotations: {
+      title: "Resolve Website Task",
+      openWorldHint: true
+    },
     inputSchema: {
       type: "object",
+      additionalProperties: false,
       properties: {
-        intent: { type: "string", description: "Plain-English description of what data to extract" },
-        url: { type: "string", description: "Target website URL" },
+        intent: { type: "string", description: "Plain-English user task, e.g. 'get feed posts' or 'find product prices'. Describe the visible goal, not the API route." },
+        url: { type: "string", description: "Concrete page URL for the task. Prefer the exact page with the needed data, not a homepage." },
         path: { type: "string", description: "Drill into a nested response path (e.g. 'data.items[]')" },
         extract: { type: "string", description: "Pick specific fields: 'field1,alias:deep.path'" },
         limit: { type: "number", description: "Cap array output to N items (1-200)" },
@@ -950,30 +983,45 @@ var TOOLS = [
         confirmUnsafe: { type: "boolean", description: "Allow non-GET requests" }
       },
       required: ["intent", "url"]
-    }
+    },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_search",
-    description: "Search the unbrowse skill marketplace for pre-built API skills. Faster than resolving from scratch if a skill already exists for the target site.",
+    title: "Search Learned Skills",
+    description: "Search the Unbrowse marketplace for an existing learned skill before triggering a new capture. Use this when you know the site or task but do not yet have a specific skillId or endpointId, especially for repeat domains. Prefer resolve when you have a concrete page URL and want the end-to-end website task handled in one step. Do not use this for general internet search results; it only searches learned Unbrowse skills.",
+    annotations: {
+      title: "Search Learned Skills",
+      readOnlyHint: true,
+      openWorldHint: true
+    },
     inputSchema: {
       type: "object",
+      additionalProperties: false,
       properties: {
         intent: { type: "string", description: "What you're looking for (e.g. 'hacker news top stories')" },
         domain: { type: "string", description: "Filter results to a specific domain" }
       },
       required: ["intent"]
-    }
+    },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_execute",
-    description: "Execute a previously discovered skill endpoint. Use after resolve or search returns a skill ID and endpoint ID.",
+    title: "Execute Learned Endpoint",
+    description: "Execute a specific Unbrowse endpoint after resolve or search has already identified the right skillId and endpointId. Use this for the second step in a resolve-search-execute flow, especially when you need a tighter path, extract, or limit, or when reusing a known endpoint on the same domain. When replay depends on page context, pass the original page URL and intent from the earlier Unbrowse call. Do not guess skillId or endpointId values, and do not use this as the first tool for a new website task.",
+    annotations: {
+      title: "Execute Learned Endpoint",
+      openWorldHint: true
+    },
     inputSchema: {
       type: "object",
+      additionalProperties: false,
       properties: {
-        skillId: { type: "string", description: "Skill ID to execute" },
-        endpointId: { type: "string", description: "Endpoint ID within the skill" },
-        url: { type: "string", description: "Optional source URL when endpoint replay needs page context" },
-        intent: { type: "string", description: "Optional original intent when endpoint replay needs selection context" },
+        skillId: { type: "string", description: "Known skill ID returned by unbrowse_resolve, unbrowse_search, or unbrowse_skill" },
+        endpointId: { type: "string", description: "Known endpoint ID inside that skill" },
+        url: { type: "string", description: "Recommended for browser-capture skills: the original page URL so replay keeps the same page and query context" },
+        intent: { type: "string", description: "Recommended for browser-capture skills: the original user intent so replay keeps the same task context" },
         path: { type: "string", description: "Drill into a nested response path" },
         extract: { type: "string", description: "Pick specific fields" },
         limit: { type: "number", description: "Cap array output to N items" },
@@ -982,39 +1030,66 @@ var TOOLS = [
         confirmUnsafe: { type: "boolean", description: "Allow non-GET requests" }
       },
       required: ["skillId", "endpointId"]
-    }
+    },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_login",
-    description: "Open a browser for the user to log into a website. Captures auth cookies so future resolve/execute calls can access authenticated content.",
+    title: "Capture Site Login",
+    description: "Open an interactive browser login flow for a gated site so later Unbrowse calls can reuse the captured auth state. Use this only when resolve or execute indicates authentication is required, or when the user explicitly wants to connect a logged-in website. Do not use this for ordinary public pages.",
+    annotations: {
+      title: "Capture Site Login",
+      openWorldHint: true
+    },
     inputSchema: {
       type: "object",
+      additionalProperties: false,
       properties: {
-        url: { type: "string", description: "Login page URL" }
+        url: { type: "string", description: "Concrete site or login page URL that needs auth cookies" }
       },
       required: ["url"]
-    }
+    },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_skills",
-    description: "List all locally cached unbrowse skills.",
-    inputSchema: { type: "object", properties: {} }
+    title: "List Cached Skills",
+    description: "Debug/admin tool. List locally cached Unbrowse skills on this machine. Use this for inspection or troubleshooting, not as the normal first step for website tasks.",
+    annotations: {
+      title: "List Cached Skills",
+      readOnlyHint: true
+    },
+    inputSchema: { type: "object", additionalProperties: false, properties: {} },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_skill",
-    description: "Get details of a specific cached skill, including its endpoints and schemas.",
+    title: "Inspect One Cached Skill",
+    description: "Debug/admin tool. Inspect one known cached Unbrowse skill, including endpoint IDs and schemas. Use this only after you already have a skillId and need to inspect it; not as the primary path for a new website task.",
+    annotations: {
+      title: "Inspect One Cached Skill",
+      readOnlyHint: true
+    },
     inputSchema: {
       type: "object",
+      additionalProperties: false,
       properties: {
-        skillId: { type: "string", description: "Skill ID to inspect" }
+        skillId: { type: "string", description: "Known skill ID returned by another Unbrowse tool" }
       },
       required: ["skillId"]
-    }
+    },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_health",
-    description: "Check if the unbrowse CLI and local server are working.",
-    inputSchema: { type: "object", properties: {} }
+    title: "Check Unbrowse Health",
+    description: "Debug/admin tool. Check whether the Unbrowse CLI and local server are installed and reachable. Use this for setup or troubleshooting, not as part of a normal website workflow.",
+    annotations: {
+      title: "Check Unbrowse Health",
+      readOnlyHint: true
+    },
+    inputSchema: { type: "object", additionalProperties: false, properties: {} },
+    outputSchema: TOOL_RESULT_SCHEMA
   }
 ];
 function toolParamsFromCall(toolName, args) {
@@ -1063,6 +1138,55 @@ function cliErrorText(stdout) {
   }
   return null;
 }
+function parseCliJson(stdout) {
+  const trimmed = stdout.trim();
+  if (!trimmed)
+    return;
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    return;
+  }
+}
+function stringifyForText(value, fallback) {
+  if (value === undefined)
+    return fallback;
+  if (typeof value === "string")
+    return value;
+  try {
+    return JSON.stringify(value, null, 2);
+  } catch {
+    return fallback;
+  }
+}
+function buildToolSuccess(toolName, stdout) {
+  const parsed = parseCliJson(stdout);
+  const trimmed = stdout.trim();
+  return {
+    content: [{ type: "text", text: stringifyForText(parsed, trimmed || "OK") }],
+    structuredContent: {
+      ok: true,
+      tool: toolName,
+      ...parsed !== undefined ? { data: parsed } : {},
+      ...trimmed ? { rawText: trimmed } : {}
+    }
+  };
+}
+function buildToolError(toolName, errorText, stdout = "") {
+  const parsed = parseCliJson(stdout);
+  const trimmed = stdout.trim();
+  return {
+    content: [{ type: "text", text: `Error: ${errorText}` }],
+    structuredContent: {
+      ok: false,
+      tool: toolName,
+      error: errorText,
+      ...parsed !== undefined ? { data: parsed } : {},
+      ...trimmed ? { rawText: trimmed } : {}
+    },
+    isError: true
+  };
+}
 async function startMcpServer(unbrowseBin) {
   const timeoutMs = Number(process.env.UNBROWSE_TIMEOUT_MS) || 120000;
   let buffer = "";
@@ -1129,20 +1253,12 @@ async function handleMessage(msg, unbrowseBin, timeoutMs) {
         const payloadError = cliErrorText(result.stdout);
         if (!result.ok || payloadError) {
           const errorText = payloadError || result.stderr?.trim() || result.stdout?.trim() || "Command failed";
-          process.stdout.write(jsonRpcResponse(id, {
-            content: [{ type: "text", text: `Error: ${errorText}` }],
-            isError: true
-          }));
+          process.stdout.write(jsonRpcResponse(id, buildToolError(toolName, errorText, result.stdout)));
         } else {
-          process.stdout.write(jsonRpcResponse(id, {
-            content: [{ type: "text", text: result.stdout.trim() || "OK" }]
-          }));
+          process.stdout.write(jsonRpcResponse(id, buildToolSuccess(toolName, result.stdout)));
         }
       } catch (err) {
-        process.stdout.write(jsonRpcResponse(id, {
-          content: [{ type: "text", text: `Error: ${err instanceof Error ? err.message : String(err)}` }],
-          isError: true
-        }));
+        process.stdout.write(jsonRpcResponse(id, buildToolError(toolName, err instanceof Error ? err.message : String(err))));
       }
       break;
     }