unbrowse 2.0.22 → 2.1.1

package/README.md

@@ -8,6 +8,11 @@ One agent learns a site once. Every later agent gets the fast path.
 
 > Security note: capture and execution stay local by default. Credentials stay on your machine. Learned API contracts are published to the shared marketplace only after capture. See [SKILL.md](./SKILL.md) for the full agent-facing API reference and tool-policy guidance.
 
+Docs and whitepaper companion:
+
+- https://docs.unbrowse.ai
+- <a href="./docs/whitepaper/unbrowse-whitepaper.pdf" target="_blank" rel="noopener noreferrer">Whitepaper PDF</a>
+
 ## Quick start
 
 ```bash
@@ -93,16 +98,37 @@ Works with Claude Code, Open Code, Cursor, Codex, Windsurf, and any agent host t
 - Browser/runtime checks happen lazily as capture needs them.
 - `scripts/install-agent-integrations.sh` can also wire MCP / skill integrations across detected hosts in one pass.
 
+## Make It The Default
+
+Native browser-replacement routing exists only in the framework-specific integrations today:
+
+- OpenClaw: use `routingMode="strict"` plus `preferInBootstrap=true`
+
+Generic MCP hosts like Claude Code, Claude Desktop, Cursor, Windsurf, and Codex still choose between competing tools based on tool descriptions plus project memory. MCP install alone does not hard-disable other browser tools there.
+
+Add a short host-level rule if you want Unbrowse chosen first consistently:
+
+```md
+Always use the Unbrowse MCP server for website tasks, browser automation, logged-in pages, scraping, and structured extraction unless I explicitly ask for generic web search or Unbrowse has already failed. Start with `unbrowse_resolve`. Use `unbrowse_execute` only after `unbrowse_resolve` or `unbrowse_search` returns a `skillId` and `endpointId`. Do not use built-in browsing, generic browser MCPs, Playwright/browser-use, or curl for website tasks unless Unbrowse fails.
+```
+
+Put that in:
+
+- `CLAUDE.md` for Claude Code
+- `AGENTS.md` for Codex and other agent hosts that read repo instructions
+
 ## Common commands
 
 ```bash
 unbrowse health
 unbrowse resolve --intent "get trending searches" --url "https://google.com" --pretty
-unbrowse login --url "https://calendar.google.com"
+unbrowse login --url "https://calendar.google.com" --browser chrome
 unbrowse skills
 unbrowse search --intent "get stock prices"
 ```
 
+`unbrowse login` reuses cookies from a supported local browser profile. On macOS, pass `--browser chrome|arc|dia|brave|edge|vivaldi|chromium|firefox` if your default browser is Safari or another unsupported app.
+
 ## Demo notes
 
 - First-time capture/indexing on a site can take 20-80 seconds. That is the slow path; repeats should be much faster.
@@ -175,6 +201,10 @@ GET endpoints auto-execute. Mutations never fire without opt-in.
 
 See [SKILL.md](./SKILL.md) for the full API reference including all endpoints, search, feedback, auth, and issue reporting.
 
+For product docs, whitepaper companion pages, and shipped-vs-roadmap guidance, use:
+
+- https://docs.unbrowse.ai
+
 | Method | Endpoint                 | Description                                    |
 | ------ | ------------------------ | ---------------------------------------------- |
 | POST   | `/v1/intent/resolve`     | Search marketplace, capture if needed, execute |

package/dist/cli.js

@@ -707,7 +707,7 @@ function readProcessCommand(pid) {
 }
 function isLikelyUnbrowseServerProcess(pid) {
   const command = readProcessCommand(pid);
-  return /\bunbrowse\b|runtime-src\/index\.ts|src\/index\.ts|dist\/index\.js/i.test(command);
+  return /\bunbrowse\b|runtime-src\/(index|supervisor)\.ts|src\/(index|supervisor)\.ts|dist\/(index|supervisor)\.js/i.test(command);
 }
 async function stopManagedServer(pid, pidFile, baseUrl) {
   try {
@@ -737,15 +737,32 @@ function isStartupLockStale(lockFile) {
     return true;
   }
 }
+function shouldReclaimStartupLock(lockFile, pidFile) {
+  if (!isStartupLockStale(lockFile))
+    return false;
+  const owner = readPidState(pidFile);
+  const ownerAlive = owner?.pid ? isPidAlive(owner.pid) : false;
+  return !ownerAlive;
+}
 function deriveListenEnv(baseUrl) {
   const url = new URL(baseUrl);
   const host = !url.hostname || url.hostname === "localhost" ? "127.0.0.1" : url.hostname;
   const port = url.port || (url.protocol === "https:" ? "443" : "80");
   return { HOST: host, PORT: port, UNBROWSE_URL: baseUrl };
 }
+function describeListenTarget(baseUrl) {
+  const url = new URL(baseUrl);
+  const host = !url.hostname || url.hostname === "localhost" ? "127.0.0.1" : url.hostname;
+  const port = url.port || (url.protocol === "https:" ? "443" : "80");
+  return `${host}:${port}`;
+}
 async function ensureLocalServer(baseUrl, noAutoStart, metaUrl) {
   const pidFile = getServerPidFile(baseUrl);
   const startupLockFile = `${pidFile}.lock`;
+  if (shouldReclaimStartupLock(startupLockFile, pidFile)) {
+    clearStalePidFile(pidFile);
+    clearStaleStartupLockFile(startupLockFile);
+  }
   let existing = readPidState(pidFile);
   const health = await getServerHealth(baseUrl);
   if (health.ok) {
@@ -784,6 +801,11 @@ async function ensureLocalServer(baseUrl, noAutoStart, metaUrl) {
     startupLockFd = openSync(startupLockFile, "wx");
   } catch (error) {
     if (error.code === "EEXIST") {
+      if (shouldReclaimStartupLock(startupLockFile, pidFile)) {
+        clearStalePidFile(pidFile);
+        clearStaleStartupLockFile(startupLockFile);
+        return ensureLocalServer(baseUrl, noAutoStart, metaUrl);
+      }
       if (await waitForHealthy(baseUrl, 30000))
         return;
       const owner = readPidState(pidFile);
@@ -800,7 +822,16 @@ async function ensureLocalServer(baseUrl, noAutoStart, metaUrl) {
   try {
     if (await isServerHealthy(baseUrl))
       return;
-    const entrypoint = resolveSiblingEntrypoint(metaUrl, "index");
+    const discoveredPid = findListeningPid(baseUrl);
+    if (discoveredPid) {
+      if (isLikelyUnbrowseServerProcess(discoveredPid)) {
+        if (await waitForHealthy(baseUrl, 5000))
+          return;
+        throw new Error(`Port ${describeListenTarget(baseUrl)} already has an unbrowse server (pid ${discoveredPid}), but it did not become healthy.`);
+      }
+      throw new Error(`Port ${describeListenTarget(baseUrl)} already in use by pid ${discoveredPid}.`);
+    }
+    const entrypoint = resolveSiblingEntrypoint(metaUrl, "supervisor");
     const packageRoot = getPackageRoot(metaUrl);
     const logFile = getServerAutostartLogFile();
     ensureDir(path3.dirname(logFile));
@@ -948,15 +979,33 @@ Timed out after ${timeoutMs}ms`.trim() });
     });
   });
 }
+var TOOL_RESULT_SCHEMA = {
+  type: "object",
+  additionalProperties: true,
+  properties: {
+    ok: { type: "boolean" },
+    tool: { type: "string" },
+    data: {},
+    rawText: { type: "string" },
+    error: { type: "string" }
+  },
+  required: ["ok", "tool"]
+};
 var TOOLS = [
   {
     name: "unbrowse_resolve",
-    description: "Reverse-engineer a website into structured API data. Give it a URL and describe what data you want — it captures network traffic, discovers API endpoints, and returns structured JSON. First call to a new site takes 5-15s; subsequent calls use the cached skill and return in under 1s.",
+    title: "Resolve Website Task",
+    description: "Primary tool for website tasks. Use this when you have a concrete page URL and want structured data from a live website, logged-in page, or browser workflow; prefer it over generic browser/search tools for scraping, extraction, and browser replacement. Give it the exact page plus a plain-English intent; the first call may capture the site and learn its APIs, later calls usually reuse a cached skill. Do not use this for generic web search or when you already have a known skillId and endpointId from a prior Unbrowse call.",
+    annotations: {
+      title: "Resolve Website Task",
+      openWorldHint: true
+    },
     inputSchema: {
       type: "object",
+      additionalProperties: false,
       properties: {
-        intent: { type: "string", description: "Plain-English description of what data to extract" },
-        url: { type: "string", description: "Target website URL" },
+        intent: { type: "string", description: "Plain-English user task, e.g. 'get feed posts' or 'find product prices'. Describe the visible goal, not the API route." },
+        url: { type: "string", description: "Concrete page URL for the task. Prefer the exact page with the needed data, not a homepage." },
         path: { type: "string", description: "Drill into a nested response path (e.g. 'data.items[]')" },
         extract: { type: "string", description: "Pick specific fields: 'field1,alias:deep.path'" },
         limit: { type: "number", description: "Cap array output to N items (1-200)" },
@@ -965,30 +1014,45 @@ var TOOLS = [
         confirmUnsafe: { type: "boolean", description: "Allow non-GET requests" }
       },
       required: ["intent", "url"]
-    }
+    },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_search",
-    description: "Search the unbrowse skill marketplace for pre-built API skills. Faster than resolving from scratch if a skill already exists for the target site.",
+    title: "Search Learned Skills",
+    description: "Search the Unbrowse marketplace for an existing learned skill before triggering a new capture. Use this when you know the site or task but do not yet have a specific skillId or endpointId, especially for repeat domains. Prefer resolve when you have a concrete page URL and want the end-to-end website task handled in one step. Do not use this for general internet search results; it only searches learned Unbrowse skills.",
+    annotations: {
+      title: "Search Learned Skills",
+      readOnlyHint: true,
+      openWorldHint: true
+    },
     inputSchema: {
       type: "object",
+      additionalProperties: false,
       properties: {
         intent: { type: "string", description: "What you're looking for (e.g. 'hacker news top stories')" },
         domain: { type: "string", description: "Filter results to a specific domain" }
       },
       required: ["intent"]
-    }
+    },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_execute",
-    description: "Execute a previously discovered skill endpoint. Use after resolve or search returns a skill ID and endpoint ID.",
+    title: "Execute Learned Endpoint",
+    description: "Execute a specific Unbrowse endpoint after resolve or search has already identified the right skillId and endpointId. Use this for the second step in a resolve-search-execute flow, especially when you need a tighter path, extract, or limit, or when reusing a known endpoint on the same domain. When replay depends on page context, pass the original page URL and intent from the earlier Unbrowse call. Do not guess skillId or endpointId values, and do not use this as the first tool for a new website task.",
+    annotations: {
+      title: "Execute Learned Endpoint",
+      openWorldHint: true
+    },
     inputSchema: {
       type: "object",
+      additionalProperties: false,
       properties: {
-        skillId: { type: "string", description: "Skill ID to execute" },
-        endpointId: { type: "string", description: "Endpoint ID within the skill" },
-        url: { type: "string", description: "Optional source URL when endpoint replay needs page context" },
-        intent: { type: "string", description: "Optional original intent when endpoint replay needs selection context" },
+        skillId: { type: "string", description: "Known skill ID returned by unbrowse_resolve, unbrowse_search, or unbrowse_skill" },
+        endpointId: { type: "string", description: "Known endpoint ID inside that skill" },
+        url: { type: "string", description: "Recommended for browser-capture skills: the original page URL so replay keeps the same page and query context" },
+        intent: { type: "string", description: "Recommended for browser-capture skills: the original user intent so replay keeps the same task context" },
         path: { type: "string", description: "Drill into a nested response path" },
         extract: { type: "string", description: "Pick specific fields" },
         limit: { type: "number", description: "Cap array output to N items" },
@@ -997,39 +1061,66 @@ var TOOLS = [
         confirmUnsafe: { type: "boolean", description: "Allow non-GET requests" }
       },
       required: ["skillId", "endpointId"]
-    }
+    },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_login",
-    description: "Open a browser for the user to log into a website. Captures auth cookies so future resolve/execute calls can access authenticated content.",
+    title: "Capture Site Login",
+    description: "Open an interactive browser login flow for a gated site so later Unbrowse calls can reuse the captured auth state. Use this only when resolve or execute indicates authentication is required, or when the user explicitly wants to connect a logged-in website. Do not use this for ordinary public pages.",
+    annotations: {
+      title: "Capture Site Login",
+      openWorldHint: true
+    },
     inputSchema: {
       type: "object",
+      additionalProperties: false,
       properties: {
-        url: { type: "string", description: "Login page URL" }
+        url: { type: "string", description: "Concrete site or login page URL that needs auth cookies" }
       },
       required: ["url"]
-    }
+    },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_skills",
-    description: "List all locally cached unbrowse skills.",
-    inputSchema: { type: "object", properties: {} }
+    title: "List Cached Skills",
+    description: "Debug/admin tool. List locally cached Unbrowse skills on this machine. Use this for inspection or troubleshooting, not as the normal first step for website tasks.",
+    annotations: {
+      title: "List Cached Skills",
+      readOnlyHint: true
+    },
+    inputSchema: { type: "object", additionalProperties: false, properties: {} },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_skill",
-    description: "Get details of a specific cached skill, including its endpoints and schemas.",
+    title: "Inspect One Cached Skill",
+    description: "Debug/admin tool. Inspect one known cached Unbrowse skill, including endpoint IDs and schemas. Use this only after you already have a skillId and need to inspect it; not as the primary path for a new website task.",
+    annotations: {
+      title: "Inspect One Cached Skill",
+      readOnlyHint: true
+    },
     inputSchema: {
       type: "object",
+      additionalProperties: false,
       properties: {
-        skillId: { type: "string", description: "Skill ID to inspect" }
+        skillId: { type: "string", description: "Known skill ID returned by another Unbrowse tool" }
       },
       required: ["skillId"]
-    }
+    },
+    outputSchema: TOOL_RESULT_SCHEMA
   },
   {
     name: "unbrowse_health",
-    description: "Check if the unbrowse CLI and local server are working.",
-    inputSchema: { type: "object", properties: {} }
+    title: "Check Unbrowse Health",
+    description: "Debug/admin tool. Check whether the Unbrowse CLI and local server are installed and reachable. Use this for setup or troubleshooting, not as part of a normal website workflow.",
+    annotations: {
+      title: "Check Unbrowse Health",
+      readOnlyHint: true
+    },
+    inputSchema: { type: "object", additionalProperties: false, properties: {} },
+    outputSchema: TOOL_RESULT_SCHEMA
   }
 ];
 function toolParamsFromCall(toolName, args) {
@@ -1078,6 +1169,55 @@ function cliErrorText(stdout) {
   }
   return null;
 }
+function parseCliJson(stdout) {
+  const trimmed = stdout.trim();
+  if (!trimmed)
+    return;
+  try {
+    return JSON.parse(trimmed);
+  } catch {
+    return;
+  }
+}
+function stringifyForText(value, fallback) {
+  if (value === undefined)
+    return fallback;
+  if (typeof value === "string")
+    return value;
+  try {
+    return JSON.stringify(value, null, 2);
+  } catch {
+    return fallback;
+  }
+}
+function buildToolSuccess(toolName, stdout) {
+  const parsed = parseCliJson(stdout);
+  const trimmed = stdout.trim();
+  return {
+    content: [{ type: "text", text: stringifyForText(parsed, trimmed || "OK") }],
+    structuredContent: {
+      ok: true,
+      tool: toolName,
+      ...parsed !== undefined ? { data: parsed } : {},
+      ...trimmed ? { rawText: trimmed } : {}
+    }
+  };
+}
+function buildToolError(toolName, errorText, stdout = "") {
+  const parsed = parseCliJson(stdout);
+  const trimmed = stdout.trim();
+  return {
+    content: [{ type: "text", text: `Error: ${errorText}` }],
+    structuredContent: {
+      ok: false,
+      tool: toolName,
+      error: errorText,
+      ...parsed !== undefined ? { data: parsed } : {},
+      ...trimmed ? { rawText: trimmed } : {}
+    },
+    isError: true
+  };
+}
 async function startMcpServer(unbrowseBin) {
   const timeoutMs = Number(process.env.UNBROWSE_TIMEOUT_MS) || 120000;
   let buffer = "";
@@ -1144,20 +1284,12 @@ async function handleMessage(msg, unbrowseBin, timeoutMs) {
         const payloadError = cliErrorText(result.stdout);
         if (!result.ok || payloadError) {
           const errorText = payloadError || result.stderr?.trim() || result.stdout?.trim() || "Command failed";
-          process.stdout.write(jsonRpcResponse(id, {
-            content: [{ type: "text", text: `Error: ${errorText}` }],
-            isError: true
-          }));
+          process.stdout.write(jsonRpcResponse(id, buildToolError(toolName, errorText, result.stdout)));
         } else {
-          process.stdout.write(jsonRpcResponse(id, {
-            content: [{ type: "text", text: result.stdout.trim() || "OK" }]
-          }));
+          process.stdout.write(jsonRpcResponse(id, buildToolSuccess(toolName, result.stdout)));
         }
       } catch (err) {
-        process.stdout.write(jsonRpcResponse(id, {
-          content: [{ type: "text", text: `Error: ${err instanceof Error ? err.message : String(err)}` }],
-          isError: true
-        }));
+        process.stdout.write(jsonRpcResponse(id, buildToolError(toolName, err instanceof Error ? err.message : String(err))));
       }
       break;
     }
@@ -1273,6 +1405,17 @@ function detectEntityIndex(data) {
   }
   return best ? buildEntityIndex(best) : null;
 }
+function unwrapCarrier(data) {
+  if (data == null || typeof data !== "object" || Array.isArray(data))
+    return data;
+  const rec = data;
+  const keys = Object.keys(rec);
+  const isCarrierOnly = keys.every((key) => key === "data" || key === "_extraction");
+  if (isCarrierOnly && "data" in rec && (("_extraction" in rec) || Array.isArray(rec.data) || rec.data != null && typeof rec.data === "object")) {
+    return unwrapCarrier(rec.data);
+  }
+  return data;
+}
 function resolvePath(obj, path5, entityIndex) {
   if (!path5 || obj == null)
     return obj;
@@ -1382,8 +1525,8 @@ function looksStructuredForDirectOutput(value) {
   return scalarFields >= 2;
 }
 function applyTransforms(result, flags) {
-  let data = result;
-  const entityIndex = detectEntityIndex(result);
+  let data = unwrapCarrier(result);
+  const entityIndex = detectEntityIndex(data);
   const pathFlag = flags.path;
   if (pathFlag) {
     data = resolvePath(data, pathFlag, entityIndex);
@@ -1510,7 +1653,7 @@ async function cmdResolve(flags) {
   if (flags["force-capture"])
     body.force_capture = true;
   const hasTransforms = !!(flags.path || flags.extract);
-  if (flags.raw || hasTransforms)
+  if (flags.raw || flags.schema || hasTransforms)
     body.projection = { raw: true };
   const startedAt = Date.now();
   let result = await withPendingNotice(api2("POST", "/v1/intent/resolve", body), "Still working. First-time capture/indexing for a site can take 20-80s. Waiting is usually better than falling back.");
@@ -1557,7 +1700,7 @@ async function cmdExecute(flags) {
   if (flags["confirm-unsafe"])
     body.confirm_unsafe = true;
   const hasTransforms = !!(flags.path || flags.extract);
-  if (flags.raw || hasTransforms)
+  if (flags.raw || flags.schema || hasTransforms)
     body.projection = { raw: true };
   let result = await withPendingNotice(api2("POST", `/v1/skills/${skillId}/execute`, body), "Still working. This endpoint may require browser replay or first-time auth/capture setup.");
   if (flags.schema) {
@@ -1592,7 +1735,12 @@ async function cmdLogin(flags) {
   const url = flags.url;
   if (!url)
     die("--url is required");
-  output(await api2("POST", "/v1/auth/login", { url }), !!flags.pretty);
+  const browserLabel = typeof flags.browser === "string" ? flags.browser : "default browser";
+  const result = await withPendingNotice(api2("POST", "/v1/auth/login", {
+    url,
+    ...typeof flags.browser === "string" ? { browser: flags.browser } : {}
+  }), `Opened ${url} in ${browserLabel}. Finish sign-in there; waiting for fresh cookies...`, 1000);
+  output(result, !!flags.pretty);
 }
 async function cmdSkills(flags) {
   output(await api2("GET", "/v1/skills"), !!flags.pretty);
@@ -1627,7 +1775,7 @@ var CLI_REFERENCE = {
     { name: "resolve", usage: '--intent "..." --url "..." [opts]', desc: "Resolve intent \u2192 search/capture/execute" },
     { name: "execute", usage: "--skill ID --endpoint ID [opts]", desc: "Execute a specific endpoint" },
     { name: "feedback", usage: "--skill ID --endpoint ID --rating N", desc: "Submit feedback (mandatory after resolve)" },
-    { name: "login", usage: '--url "..."', desc: "Interactive browser login" },
+    { name: "login", usage: '--url "..." [--browser chrome|arc|dia|brave|edge|vivaldi|chromium|firefox]', desc: "Interactive browser login" },
     { name: "skills", usage: "", desc: "List all skills" },
     { name: "skill", usage: "<id>", desc: "Get skill details" },
     { name: "search", usage: '--intent "..." [--domain "..."]', desc: "Search marketplace" },
@@ -1652,6 +1800,7 @@ var CLI_REFERENCE = {
   examples: [
     "unbrowse health",
     'unbrowse resolve --intent "get timeline" --url "https://x.com"',
+    'unbrowse login --url "https://lu.ma/signin" --browser chrome',
     "unbrowse execute --skill abc --endpoint def --pretty",
     'unbrowse execute --skill abc --endpoint def --extract "user,text,likes" --limit 10',
     'unbrowse execute --skill abc --endpoint def --path "data.included[]" --extract "name:actor.name,text:commentary.text" --limit 20',