unbrowse 2.0.0 → 2.0.2

package/README.md

@@ -15,7 +15,7 @@ One agent learns a site once. Every later agent gets the fast path.
 npx unbrowse setup
 ```
 
-`npx unbrowse setup` downloads the CLI on demand, installs browser assets, lets you register with an email-shaped display identity, registers the Open Code `/unbrowse` command when Open Code is detected, and starts the local server.
+`npx unbrowse setup` downloads the CLI on demand, verifies the bundled Kuri runtime, lets you register with an email-shaped display identity, registers the Open Code `/unbrowse` command when Open Code is detected, and starts the local server.
 
 For daily use:
 
@@ -30,6 +30,25 @@ If your agent host uses skills:
 npx skills add unbrowse-ai/unbrowse
 ```
 
+## Upgrading
+
+Unbrowse no longer self-updates at runtime. If you already have Unbrowse installed, upgrade to the latest version after each release or the new flow may not work on your machine.
+
+If you installed the CLI globally:
+
+```bash
+npm install -g unbrowse@latest
+unbrowse setup
+```
+
+If your agent host uses skills, rerun its skill install/update command too:
+
+```bash
+npx skills add unbrowse-ai/unbrowse
+```
+
+Need help or want release updates? Join the Discord: [discord.gg/VWugEeFNsG](https://discord.gg/VWugEeFNsG)
+
 Every CLI command auto-starts the local server on `http://localhost:6969` by default. Override with `UNBROWSE_URL`, `PORT`, or `HOST`. On first startup it auto-registers as an agent with the marketplace and caches credentials in `~/.unbrowse/config.json`. `unbrowse setup` now prompts for an email-shaped identity first; headless setups can provide `UNBROWSE_AGENT_EMAIL`.
 
 Works with Claude Code, Open Code, Cursor, Codex, Windsurf, and any agent host that can call a local CLI or skill.
@@ -37,7 +56,7 @@ Works with Claude Code, Open Code, Cursor, Codex, Windsurf, and any agent host t
 ## What setup does
 
 - Checks local prerequisites for the npm/npx flow.
-- Installs browser assets needed for live capture.
+- Verifies the bundled Kuri binary, or builds it from the vendored Kuri source when working from repo source with Zig installed.
 - Registers the Open Code `/unbrowse` command when Open Code is present.
 - Starts the local Unbrowse server unless `--no-start` is passed.
 

package/dist/cli.js

@@ -1,23 +1,5 @@
 #!/usr/bin/env node
 // @bun
-import { createRequire } from "node:module";
-var __create = Object.create;
-var __getProtoOf = Object.getPrototypeOf;
-var __defProp = Object.defineProperty;
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __hasOwnProp = Object.prototype.hasOwnProperty;
-var __toESM = (mod, isNodeMode, target) => {
-  target = mod != null ? __create(__getProtoOf(mod)) : {};
-  const to = isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target;
-  for (let key of __getOwnPropNames(mod))
-    if (!__hasOwnProp.call(to, key))
-      __defProp(to, key, {
-        get: () => mod[key],
-        enumerable: true
-      });
-  return to;
-};
-var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/cli.ts
 import { config as loadEnv } from "dotenv";
@@ -322,7 +304,7 @@ import { spawn } from "node:child_process";
 import { existsSync as existsSync2, mkdirSync as mkdirSync2, realpathSync } from "node:fs";
 import os from "node:os";
 import path from "node:path";
-import { createRequire as createRequire2 } from "node:module";
+import { createRequire } from "node:module";
 import { fileURLToPath } from "node:url";
 function getModuleDir(metaUrl) {
   return path.dirname(fileURLToPath(metaUrl));
@@ -344,7 +326,7 @@ function runtimeArgsForEntrypoint(metaUrl, entrypoint) {
   if (process.versions.bun)
     return [entrypoint];
   try {
-    const req = createRequire2(metaUrl);
+    const req = createRequire(metaUrl);
     const tsxPkg = req.resolve("tsx/package.json");
     const tsxLoader = path.join(path.dirname(tsxPkg), "dist", "loader.mjs");
     if (existsSync2(tsxLoader))
@@ -483,16 +465,88 @@ function isMainModule(metaUrl) {
 }
 
 // ../../src/runtime/setup.ts
-import { execFileSync } from "node:child_process";
-import { createRequire as createRequire3 } from "node:module";
-import { existsSync as existsSync4, mkdirSync as mkdirSync4, writeFileSync as writeFileSync3 } from "node:fs";
-import os2 from "node:os";
+import { execFileSync as execFileSync2 } from "node:child_process";
+import { existsSync as existsSync5, mkdirSync as mkdirSync4, writeFileSync as writeFileSync3 } from "node:fs";
+import os3 from "node:os";
+import path6 from "node:path";
+
+// ../../src/kuri/client.ts
+import { execFileSync, spawn as spawn2 } from "node:child_process";
+import { existsSync as existsSync4 } from "node:fs";
+import path5 from "node:path";
+
+// ../../src/logger.ts
 import path4 from "node:path";
-var req = createRequire3(import.meta.url);
+import os2 from "node:os";
+var LOG_DIR = path4.join(os2.homedir(), ".unbrowse", "logs");
+
+// ../../src/kuri/client.ts
+function kuriBinaryName() {
+  return process.platform === "win32" ? "kuri.exe" : "kuri";
+}
+function currentBundledKuriTarget() {
+  if (process.platform === "darwin" && process.arch === "arm64")
+    return "darwin-arm64";
+  if (process.platform === "darwin" && process.arch === "x64")
+    return "darwin-x64";
+  if (process.platform === "linux" && process.arch === "arm64")
+    return "linux-arm64";
+  if (process.platform === "linux" && process.arch === "x64")
+    return "linux-x64";
+  return null;
+}
+function resolveBinaryOnPath(name) {
+  const checker = process.platform === "win32" ? "where" : "which";
+  try {
+    const output = execFileSync(checker, [name], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+    const match = output.split(/\r?\n/).map((line) => line.trim()).find(Boolean);
+    return match || null;
+  } catch {
+    return null;
+  }
+}
+function addCandidate(candidates, candidate) {
+  if (!candidate)
+    return;
+  if (!candidates.includes(candidate))
+    candidates.push(candidate);
+}
+function getKuriSourceCandidates() {
+  const packageRoot = getPackageRoot(import.meta.url);
+  const candidates = [];
+  addCandidate(candidates, path5.join(packageRoot, "vendor", "kuri-src"));
+  addCandidate(candidates, path5.join(packageRoot, "submodules", "kuri"));
+  if (process.env.KURI_PATH)
+    addCandidate(candidates, process.env.KURI_PATH);
+  if (process.env.HOME)
+    addCandidate(candidates, path5.join(process.env.HOME, "kuri"));
+  return candidates;
+}
+function getKuriBinaryCandidates() {
+  const packageRoot = getPackageRoot(import.meta.url);
+  const binaryName = kuriBinaryName();
+  const target = currentBundledKuriTarget();
+  const candidates = [];
+  if (target)
+    addCandidate(candidates, path5.join(packageRoot, "vendor", "kuri", target, binaryName));
+  for (const sourceDir of getKuriSourceCandidates()) {
+    addCandidate(candidates, path5.join(sourceDir, "zig-out", "bin", binaryName));
+  }
+  addCandidate(candidates, resolveBinaryOnPath("kuri"));
+  return candidates;
+}
+function findKuriBinary() {
+  if (process.env.KURI_BIN)
+    return process.env.KURI_BIN;
+  const candidates = getKuriBinaryCandidates();
+  return candidates.find((candidate) => existsSync4(candidate)) ?? candidates[0] ?? kuriBinaryName();
+}
+
+// ../../src/runtime/setup.ts
 function hasBinary(name) {
   const checker = process.platform === "win32" ? "where" : "which";
   try {
-    execFileSync(checker, [name], { stdio: "ignore" });
+    execFileSync2(checker, [name], { stdio: "ignore" });
     return true;
   } catch {
     return false;
@@ -508,18 +562,18 @@ function detectPackageManagers() {
 }
 function resolveConfigHome() {
   if (process.platform === "win32") {
-    return process.env.APPDATA || path4.join(os2.homedir(), "AppData", "Roaming");
+    return process.env.APPDATA || path6.join(os3.homedir(), "AppData", "Roaming");
   }
-  return process.env.XDG_CONFIG_HOME || path4.join(os2.homedir(), ".config");
+  return process.env.XDG_CONFIG_HOME || path6.join(os3.homedir(), ".config");
 }
 function getOpenCodeGlobalCommandsDir() {
-  return path4.join(resolveConfigHome(), "opencode", "commands");
+  return path6.join(resolveConfigHome(), "opencode", "commands");
 }
 function getOpenCodeProjectCommandsDir(cwd) {
-  return path4.join(cwd, ".opencode", "commands");
+  return path6.join(cwd, ".opencode", "commands");
 }
 function detectOpenCode(cwd) {
-  return hasBinary("opencode") || existsSync4(path4.join(resolveConfigHome(), "opencode")) || existsSync4(path4.join(cwd, ".opencode"));
+  return hasBinary("opencode") || existsSync5(path6.join(resolveConfigHome(), "opencode")) || existsSync5(path6.join(cwd, ".opencode"));
 }
 function renderOpenCodeCommand() {
   return `---
@@ -547,12 +601,12 @@ function writeOpenCodeCommand(scope, cwd) {
   if (scope === "auto" && !detected) {
     return { detected: false, action: "not-detected", scope: "off" };
   }
-  const resolvedScope = scope === "project" ? "project" : scope === "global" ? "global" : existsSync4(path4.join(cwd, ".opencode")) ? "project" : "global";
+  const resolvedScope = scope === "project" ? "project" : scope === "global" ? "global" : existsSync5(path6.join(cwd, ".opencode")) ? "project" : "global";
   const commandsDir = resolvedScope === "project" ? getOpenCodeProjectCommandsDir(cwd) : getOpenCodeGlobalCommandsDir();
-  const commandFile = path4.join(ensureDir(commandsDir), "unbrowse.md");
+  const commandFile = path6.join(ensureDir(commandsDir), "unbrowse.md");
   const content = renderOpenCodeCommand();
-  const action = existsSync4(commandFile) ? "updated" : "installed";
-  mkdirSync4(path4.dirname(commandFile), { recursive: true });
+  const action = existsSync5(commandFile) ? "updated" : "installed";
+  mkdirSync4(path6.dirname(commandFile), { recursive: true });
   writeFileSync3(commandFile, content);
   return {
     detected: detected || scope !== "auto",
@@ -562,17 +616,44 @@ function writeOpenCodeCommand(scope, cwd) {
   };
 }
 async function ensureBrowserEngineInstalled() {
+  const binary = findKuriBinary();
+  if (existsSync5(binary)) {
+    return { installed: true, action: "already-installed" };
+  }
+  const sourceDir = getKuriSourceCandidates().find((candidate) => existsSync5(path6.join(candidate, "build.zig")));
+  if (!sourceDir) {
+    return {
+      installed: false,
+      action: "failed",
+      message: `Kuri binary not found. Checked ${binary}`
+    };
+  }
+  if (!hasBinary("zig")) {
+    return {
+      installed: false,
+      action: "failed",
+      message: `Kuri source found at ${sourceDir}, but Zig is not installed`
+    };
+  }
   try {
-    const { chromium } = await import("playwright-core");
-    if (existsSync4(chromium.executablePath())) {
-      return { installed: true, action: "already-installed" };
-    }
-    const agentBrowserBin = req.resolve("agent-browser/bin/agent-browser.js");
-    execFileSync(process.execPath, [agentBrowserBin, "install"], {
+    execFileSync2("zig", ["build", "-Doptimize=ReleaseFast"], {
+      cwd: sourceDir,
       stdio: "inherit",
       timeout: 300000
     });
-    return { installed: true, action: "installed" };
+    const builtBinary = findKuriBinary();
+    if (existsSync5(builtBinary)) {
+      return {
+        installed: true,
+        action: "installed",
+        message: `Built Kuri from ${sourceDir}`
+      };
+    }
+    return {
+      installed: false,
+      action: "failed",
+      message: `Kuri build completed but ${builtBinary} was not created`
+    };
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
     return { installed: false, action: "failed", message };
@@ -584,7 +665,7 @@ async function runSetup(options) {
   return {
     os: {
       platform: process.platform,
-      release: os2.release(),
+      release: os3.release(),
       arch: process.arch
     },
     package_managers: detectPackageManagers(),
@@ -621,8 +702,8 @@ function parseArgs(argv) {
   }
   return { command, args: positional, flags };
 }
-async function api2(method, path5, body) {
-  const res = await fetch(`${BASE_URL}${path5}`, {
+async function api2(method, path7, body) {
+  const res = await fetch(`${BASE_URL}${path7}`, {
     method,
     headers: {
       ...body ? { "Content-Type": "application/json" } : {},
@@ -708,10 +789,10 @@ function detectEntityIndex(data) {
   }
   return best ? buildEntityIndex(best) : null;
 }
-function resolvePath(obj, path5, entityIndex) {
-  if (!path5 || obj == null)
+function resolvePath(obj, path7, entityIndex) {
+  if (!path7 || obj == null)
     return obj;
-  const segments = path5.split(".");
+  const segments = path7.split(".");
   let cur = obj;
   for (let i = 0;i < segments.length; i++) {
     if (cur == null)
@@ -750,8 +831,8 @@ function extractFields(data, fields, entityIndex) {
     for (const f of fields) {
       const colonIdx = f.indexOf(":");
       const alias = colonIdx >= 0 ? f.slice(0, colonIdx) : f.split(".").pop();
-      const path5 = colonIdx >= 0 ? f.slice(colonIdx + 1) : f;
-      const resolved = resolvePath(item, path5, entityIndex ?? undefined) ?? [];
+      const path7 = colonIdx >= 0 ? f.slice(colonIdx + 1) : f;
+      const resolved = resolvePath(item, path7, entityIndex ?? undefined) ?? [];
       out[alias] = Array.isArray(resolved) ? resolved.length === 0 ? null : resolved.length === 1 ? resolved[0] : resolved : resolved;
     }
     return out;
@@ -1039,11 +1120,11 @@ async function cmdSearch(flags) {
   if (!intent)
     die("--intent is required");
   const domain = flags.domain;
-  const path5 = domain ? "/v1/search/domain" : "/v1/search";
+  const path7 = domain ? "/v1/search/domain" : "/v1/search";
   const body = { intent, k: Number(flags.k) || 5 };
   if (domain)
     body.domain = domain;
-  output(await api2("POST", path5, body), !!flags.pretty);
+  output(await api2("POST", path7, body), !!flags.pretty);
 }
 async function cmdSessions(flags) {
   const domain = flags.domain;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "2.0.0",
+  "version": "2.0.2",
   "description": "Reverse-engineer any website into reusable API skills. npm CLI + local engine.",
   "type": "module",
   "bin": {
@@ -9,6 +9,7 @@
   "files": [
     "dist",
     "runtime-src",
+    "vendor/kuri",
     "README.md",
     "LICENSE"
   ],
@@ -25,8 +26,6 @@
     "cheerio": "^1.2.0",
     "dotenv": "^17.3.1",
     "nanoid": "^5.1.6",
-    "agent-browser": "^0.13.0",
-    "playwright-core": "^1.58.2",
     "tsx": "^4.20.6",
     "ws": "^8.19.0"
   },

package/runtime-src/capture/index.ts

@@ -137,7 +137,9 @@ export function isBlockedAppShell(html?: string): boolean {
     /switch to a supported browser/i.test(html) ||
     /Something went wrong, but don.?t fret/i.test(html) ||
     /class=["']errorContainer["']/i.test(html) ||
-    /#placeholder,\s*#react-root\s*\{\s*display:\s*none/i.test(html)
+    /#placeholder,\s*#react-root\s*\{\s*display:\s*none/i.test(html) ||
+    /Attention Required!\s*\|\s*Cloudflare/i.test(html) ||
+    /cf-error-details|cf\.errors\.css/i.test(html)
   );
 }
 
@@ -436,7 +438,15 @@ async function waitForContentReady(
   responseBodies?: Map<string, string>,
 ): Promise<void> {
   // Phase 1: Initial settle — let the page start rendering
-  await new Promise((r) => setTimeout(r, 2000));
+  await new Promise((r) => setTimeout(r, 1000));
+
+  // Early exit: if interceptor already captured API responses, page is loaded enough
+  if (responseBodies && responseBodies.size > 0) {
+    log("capture", `early exit: ${responseBodies.size} API responses already captured during navigation`);
+    // Brief extra settle to catch any trailing responses
+    await new Promise((r) => setTimeout(r, 500));
+    return;
+  }
 
   // Phase 2: Cloudflare challenge detection and wait
   try {
@@ -453,7 +463,21 @@ async function waitForContentReady(
   }
 
   // Phase 3: Wait for document ready state (replaces networkidle)
-  await waitForReadyState(tabId, 8000);
+  await waitForReadyState(tabId, 5000);
+
+  // Early exit: check again after readyState — SPAs often fire API calls during hydration
+  if (responseBodies) {
+    const intercepted = await collectInterceptedRequests(tabId);
+    for (const entry of intercepted) {
+      if (entry.response_body && !entry.is_js) {
+        responseBodies.set(entry.url, entry.response_body);
+      }
+    }
+    if (responseBodies.size > 0) {
+      log("capture", `early exit after readyState: ${responseBodies.size} API responses captured`);
+      return;
+    }
+  }
 
   // Phase 4: Intent-aware API wait — poll intercepted requests for matching API URLs
   if (captureUrl && responseBodies) {
@@ -464,8 +488,8 @@ async function waitForContentReady(
     if (wantedHints.length > 0) {
       log("capture", `intent-aware wait: looking for API matching one of [${wantedHints.join(", ")}] (from ${captureUrl})`);
       const intentStart = Date.now();
-      const INTENT_MAX_WAIT = 15000;
-      const INTENT_POLL_INTERVAL = 1500;
+      const INTENT_MAX_WAIT = 8000;
+      const INTENT_POLL_INTERVAL = 1000;
       while (Date.now() - intentStart < INTENT_MAX_WAIT) {
         await new Promise((r) => setTimeout(r, INTENT_POLL_INTERVAL));
         // Check newly intercepted requests
@@ -505,7 +529,7 @@ async function waitForContentReady(
       await new Promise((r) => setTimeout(r, 1200));
       await kuri.evaluate(tabId, "window.scrollTo(0, 0)");
       if (responseBodies.size === before) {
-        await new Promise((r) => setTimeout(r, 2000));
+        await new Promise((r) => setTimeout(r, 1500));
       }
     } catch {
       // non-fatal
@@ -638,14 +662,9 @@ export async function captureSession(
     try { pageDomain = getRegistrableDomain(new URL(url).hostname); } catch { /* bad url */ }
 
     // Inject fetch/XHR interceptor BEFORE navigation to capture all response bodies
-    // Navigate to origin first so the interceptor runs in the correct context
-    try {
-      const origin = new URL(url).origin;
-      await kuri.navigate(tabId, origin);
-      await new Promise((r) => setTimeout(r, 500));
-    } catch { /* best-effort */ }
-
-    await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT);
+    // Navigate directly to target URL — skip origin pre-navigation to save 1-2s on heavy SPAs.
+    // The interceptor is re-injected after navigation anyway (page context resets on navigate).
+    await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT).catch(() => {});
 
     // Navigate to target URL
     await kuri.navigate(tabId, url);
@@ -707,10 +726,14 @@ export async function captureSession(
       log("capture", `response body captured: ${bodyUrl.substring(0, 150)}`);
     }
 
+
     let final_url = url;
     let html: string | undefined;
     try {
-      final_url = await kuri.getCurrentUrl(tabId);
+      const rawUrl = await kuri.getCurrentUrl(tabId);
+      final_url = typeof rawUrl === "string" ? rawUrl : String(rawUrl ?? url);
+      // Validate it's actually a URL, fall back to original if not
+      try { new URL(final_url); } catch { final_url = url; }
       html = await kuri.getPageHtml(tabId);
     } catch {}
 
@@ -779,6 +802,14 @@ export async function captureSession(
       responseBodyCount < 10 &&
       !hasUsefulCapturedResponses(responseBodies.keys(), url, intent)
     ) {
+      // On ephemeral retry, if still blocked by Cloudflare WAF, throw auth_required
+      // so the caller can surface a login prompt instead of retrying forever
+      if (options?.forceEphemeral && html && /Cloudflare|cf\.errors\.css|cf-error-details/i.test(html)) {
+        throw Object.assign(new Error("cloudflare_waf_block"), {
+          code: "auth_required",
+          login_url: url,
+        });
+      }
       retryFreshTab = true;
       log("capture", `rendered blocked app shell for ${url}; retrying with fresh tab`);
     } else {
@@ -807,7 +838,7 @@ export async function captureSession(
     await resetTab(tabId);
     releaseTabSlot(tabId);
   }
-  if (retryFreshTab) {
+  if (retryFreshTab && !options?.forceEphemeral) {
     return captureSession(url, authHeaders, cookies, intent, { forceEphemeral: true });
   }
   if (captureError) throw captureError;

package/runtime-src/execution/index.ts

@@ -939,7 +939,7 @@ async function executeBrowserCapture(
     skill.endpoints.find((endpoint) => typeof endpoint.trigger_url === "string" && endpoint.trigger_url)?.trigger_url ||
     skill.endpoints.find((endpoint) => !/\{[^}]+\}/.test(endpoint.url_template))?.url_template ||
     "";
-  const url = String(params.url ?? fallbackUrl);
+  const url = typeof params.url === "string" ? params.url : String(params.url ?? fallbackUrl);
   const intent = String(params.intent ?? skill.intent_signature);
   if (!url) throw new Error("browser-capture skill requires params.url");
 
@@ -981,7 +981,33 @@ async function executeBrowserCapture(
     usedStoredAuth,
   );
   if (documentSeed) return documentSeed;
-  const captured = await captureSession(url, authHeaders, cookies, intent);
+  let captured;
+  try {
+    captured = await captureSession(url, authHeaders, cookies, intent);
+  } catch (captureErr: unknown) {
+    const err = captureErr as Error & { code?: string; login_url?: string };
+    if (err.code === "auth_required") {
+      const trace: ExecutionTrace = stampTrace({
+        trace_id: traceId,
+        skill_id: skill.skill_id,
+        endpoint_id: "browser-capture",
+        started_at: startedAt,
+        completed_at: new Date().toISOString(),
+        success: false,
+        error: "auth_required",
+      });
+      return {
+        trace,
+        result: {
+          error: "auth_required",
+          provider: "cloudflare",
+          login_url: err.login_url ?? url,
+          message: `Site is blocked by Cloudflare WAF. Run: unbrowse login --url "${url}" to authenticate interactively.`,
+        },
+      };
+    }
+    throw captureErr;
+  }
 
   const finalDomain = (() => {
     try { return new URL(captured.final_url).hostname; } catch { return targetDomain; }
@@ -990,7 +1016,7 @@ async function executeBrowserCapture(
   const LOGIN_PATHS = /\/(login|signin|sign-in|sso|auth|uas\/login|checkpoint|oauth)/i;
 
   const redirectedToAuth = finalDomain !== targetDomain && AUTH_PROVIDERS.test(finalDomain);
-  const redirectedToLogin = captured.final_url !== url && LOGIN_PATHS.test(new URL(captured.final_url).pathname);
+  const redirectedToLogin = captured.final_url !== url && (() => { try { return LOGIN_PATHS.test(new URL(String(captured.final_url)).pathname); } catch { return false; } })();
 
   if (redirectedToAuth || redirectedToLogin) {
     const trace: ExecutionTrace = stampTrace({

package/runtime-src/extraction/index.ts

@@ -1278,18 +1278,38 @@ export function extractFromDOMWithHint(
  * the best match for the given intent.
  */
 export function extractFromDOM(html: string, intent: string): ExtractionResult {
+  // Cap HTML size to prevent cheerio from hanging on massive pages
+  const MAX_HTML_SIZE = 300_000;
+  let workingHtml = html;
+  if (workingHtml.length > MAX_HTML_SIZE) {
+    // Strip attribute bloat first (class/style/data-* attributes inflate HTML 2-3x)
+    workingHtml = workingHtml
+      .replace(/\s+class="[^"]*"/g, "")
+      .replace(/\s+style="[^"]*"/g, "")
+      .replace(/\s+data-[a-z][-a-z]*="[^"]*"/g, "");
+    // If still too large, truncate keeping body content
+    if (workingHtml.length > MAX_HTML_SIZE) {
+      const bodyStart = workingHtml.indexOf("<body");
+      if (bodyStart > 0) {
+        workingHtml = workingHtml.substring(0, Math.max(MAX_HTML_SIZE, bodyStart + MAX_HTML_SIZE));
+      } else {
+        workingHtml = workingHtml.substring(0, MAX_HTML_SIZE);
+      }
+    }
+  }
+
   // Extract SPA-embedded data from raw HTML BEFORE cleanDOM strips scripts
-  const spaStructures = extractSPAData(html);
-  const flashStructures = extractFlashNoticeSpecial(html, intent);
-  const cleaned = cleanDOM(html);
-  const githubStructures = extractGitHubSpecial(html, intent);
-  const linkedInStructures = extractLinkedInSpecial(html, intent);
-  const packageSearchStructures = extractPackageSearchSpecial(html, intent);
-  const xProfileStructures = extractXProfileSpecial(html, intent);
-  const postStructures = extractPostSpecial(html, intent);
-  const trendStructures = extractTrendSpecial(html, intent);
-  const definitionStructures = extractDefinitionSpecial(html, intent);
-  const courseStructures = extractCourseSearchSpecial(html, intent);
+  const spaStructures = extractSPAData(workingHtml);
+  const flashStructures = extractFlashNoticeSpecial(workingHtml, intent);
+  const cleaned = cleanDOM(workingHtml);
+  const githubStructures = extractGitHubSpecial(workingHtml, intent);
+  const linkedInStructures = extractLinkedInSpecial(workingHtml, intent);
+  const packageSearchStructures = extractPackageSearchSpecial(workingHtml, intent);
+  const xProfileStructures = extractXProfileSpecial(workingHtml, intent);
+  const postStructures = extractPostSpecial(workingHtml, intent);
+  const trendStructures = extractTrendSpecial(workingHtml, intent);
+  const definitionStructures = extractDefinitionSpecial(workingHtml, intent);
+  const courseStructures = extractCourseSearchSpecial(workingHtml, intent);
   const structures = [...flashStructures, ...githubStructures, ...linkedInStructures, ...packageSearchStructures, ...xProfileStructures, ...postStructures, ...trendStructures, ...definitionStructures, ...courseStructures, ...spaStructures, ...parseStructured(cleaned)]
     .map((structure) => normalizeStructureForIntent(structure, intent));
 
@@ -1306,7 +1326,17 @@ export function extractFromDOM(html: string, intent: string): ExtractionResult {
 
   scored.sort((a, b) => b.score - a.score);
 
-  const bestPassing = scored.find((candidate) => assessIntentResult(candidate.structure.data, intent).verdict === "pass");
+  const passing = scored.filter((candidate) => assessIntentResult(candidate.structure.data, intent).verdict === "pass");
+  const bestPassing = (() => {
+    if (passing.length === 0) return undefined;
+    const bestPassingOverall = passing[0];
+    const bestPassingSpa = passing.find((candidate) => candidate.structure.type.startsWith("spa-"));
+    // Prefer cleaner SPA payloads when they're effectively tied with DOM-derived candidates.
+    if (bestPassingSpa && bestPassingOverall && bestPassingSpa.score >= bestPassingOverall.score - 2) {
+      return bestPassingSpa;
+    }
+    return bestPassingOverall;
+  })();
   if (bestPassing) {
     return {
       data: bestPassing.structure.data,
@@ -1325,7 +1355,17 @@ export function extractFromDOM(html: string, intent: string): ExtractionResult {
       selector: best.structure.selector,
     };
   }
-  const hasClearWinner = scored.length === 1 || best.score > scored[1].score * 1.5;
+
+  if (scored.length === 1) {
+    return {
+      data: best.structure.data,
+      extraction_method: best.structure.type,
+      confidence: computeConfidence(best.structure, best.score),
+      selector: best.structure.selector,
+    };
+  }
+
+  const hasClearWinner = best.score > scored[1].score * 1.5;
 
   if (hasClearWinner && best.score > 0) {
     return {

package/runtime-src/kuri/client.ts

@@ -8,8 +8,11 @@
  * All browser ops go through HTTP — no Playwright, no Node CDP bindings.
  */
 
-import { spawn, type ChildProcess } from "node:child_process";
+import { execFileSync, spawn, type ChildProcess } from "node:child_process";
+import { existsSync } from "node:fs";
+import path from "node:path";
 import { log } from "../logger.js";
+import { getPackageRoot } from "../runtime/paths.js";
 
 const KURI_DEFAULT_PORT = 7700;
 const KURI_STARTUP_TIMEOUT_MS = 10_000;
@@ -52,6 +55,58 @@ let kuriPort = KURI_DEFAULT_PORT;
 let kuriCdpPort: number | null = null;
 let kuriReady = false;
 
+function kuriBinaryName(): string {
+  return process.platform === "win32" ? "kuri.exe" : "kuri";
+}
+
+function currentBundledKuriTarget(): string | null {
+  if (process.platform === "darwin" && process.arch === "arm64") return "darwin-arm64";
+  if (process.platform === "darwin" && process.arch === "x64") return "darwin-x64";
+  if (process.platform === "linux" && process.arch === "arm64") return "linux-arm64";
+  if (process.platform === "linux" && process.arch === "x64") return "linux-x64";
+  return null;
+}
+
+function resolveBinaryOnPath(name: string): string | null {
+  const checker = process.platform === "win32" ? "where" : "which";
+  try {
+    const output = execFileSync(checker, [name], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+    const match = output.split(/\r?\n/).map((line) => line.trim()).find(Boolean);
+    return match || null;
+  } catch {
+    return null;
+  }
+}
+
+function addCandidate(candidates: string[], candidate?: string | null): void {
+  if (!candidate) return;
+  if (!candidates.includes(candidate)) candidates.push(candidate);
+}
+
+export function getKuriSourceCandidates(): string[] {
+  const packageRoot = getPackageRoot(import.meta.url);
+  const candidates: string[] = [];
+  addCandidate(candidates, path.join(packageRoot, "vendor", "kuri-src"));
+  addCandidate(candidates, path.join(packageRoot, "submodules", "kuri"));
+  if (process.env.KURI_PATH) addCandidate(candidates, process.env.KURI_PATH);
+  if (process.env.HOME) addCandidate(candidates, path.join(process.env.HOME, "kuri"));
+  return candidates;
+}
+
+export function getKuriBinaryCandidates(): string[] {
+  const packageRoot = getPackageRoot(import.meta.url);
+  const binaryName = kuriBinaryName();
+  const target = currentBundledKuriTarget();
+  const candidates: string[] = [];
+
+  if (target) addCandidate(candidates, path.join(packageRoot, "vendor", "kuri", target, binaryName));
+  for (const sourceDir of getKuriSourceCandidates()) {
+    addCandidate(candidates, path.join(sourceDir, "zig-out", "bin", binaryName));
+  }
+  addCandidate(candidates, resolveBinaryOnPath("kuri"));
+  return candidates;
+}
+
 /** Try common CDP ports to find where Chrome is listening. */
 async function discoverCdpPort(): Promise<void> {
   const portsToTry = [9222, 9223, 9224, 9225];
@@ -128,10 +183,10 @@ async function kuriPost(path: string, params: Record<string, string>, body: unkn
 }
 
 /** Find the kuri binary — check env, then common build locations. */
-function findKuriBinary(): string {
+export function findKuriBinary(): string {
   if (process.env.KURI_BIN) return process.env.KURI_BIN;
-  const kuriRepoPath = process.env.KURI_PATH ?? `${process.env.HOME}/kuri`;
-  return `${kuriRepoPath}/zig-out/bin/kuri`;
+  const candidates = getKuriBinaryCandidates();
+  return candidates.find((candidate) => existsSync(candidate)) ?? candidates[0] ?? kuriBinaryName();
 }
 
 /**
@@ -160,6 +215,9 @@ export async function start(port?: number): Promise<void> {
 
   const binary = findKuriBinary();
   log("kuri", `starting: ${binary} on port ${kuriPort}`);
+  if (!existsSync(binary)) {
+    throw new Error(`Kuri binary not found at ${binary}`);
+  }
 
   // Check if Chrome is already running — if so, pass CDP_URL to connect
   // If not, omit CDP_URL so Kuri launches its own managed Chrome
@@ -281,12 +339,16 @@ export async function getDefaultTab(): Promise<string> {
   throw new Error("No tabs available and failed to create one");
 }
 
+/** Trigger Kuri's /discover to sync Chrome tabs into Kuri's registry. */
 /** Trigger Kuri's /discover to sync Chrome tabs into Kuri's registry. */
 async function ensureTabsDiscovered(): Promise<void> {
   try {
-    await kuriGet("/discover");
+    // Pass CDP URL as query param so /discover works even if Kuri was started without CDP_URL env
+    const params: Record<string, string> = {};
+    if (kuriCdpPort) params.cdp_url = `ws://127.0.0.1:${kuriCdpPort}`;
+    await kuriGet("/discover", params);
   } catch {
-    // /discover may fail if CDP_URL not set — that's handled by start()
+    // /discover may fail if no Chrome running — that's OK
   }
 }
 
@@ -295,12 +357,35 @@ export async function navigate(tabId: string, url: string): Promise<void> {
   await kuriGet("/navigate", { tab_id: tabId, url });
 }
 
+/** Evaluate JavaScript in tab context. */
+/** Evaluate JavaScript in tab context. */
+/** Evaluate JavaScript in tab context. */
 /** Evaluate JavaScript in tab context. */
 export async function evaluate(tabId: string, expression: string): Promise<unknown> {
-  const raw = (await kuriGet("/evaluate", { tab_id: tabId, expression })) as {
+  let raw: {
     id?: number;
     result?: { result?: { type?: string; value?: unknown; description?: string }; exceptionDetails?: unknown };
   };
+  if (expression.length > 2000) {
+    // Use POST with raw text body for large expressions to avoid URL length limits
+    const url = kuriUrl("/evaluate", { tab_id: tabId });
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), KURI_REQUEST_TIMEOUT_MS);
+    try {
+      const res = await fetch(url, {
+        method: "POST",
+        headers: { "Content-Type": "text/plain" },
+        body: expression,
+        signal: controller.signal,
+      });
+      const text = await res.text();
+      try { raw = JSON.parse(text); } catch { raw = text as never; }
+    } finally {
+      clearTimeout(timeout);
+    }
+  } else {
+    raw = (await kuriGet("/evaluate", { tab_id: tabId, expression })) as typeof raw;
+  }
   // CDP Runtime.evaluate response: { id, result: { result: { type, value } } }
   const inner = raw?.result?.result;
   if (!inner) return raw;
@@ -425,7 +510,10 @@ export async function hasCloudflareChallenge(tabId: string): Promise<boolean> {
     var html = document.documentElement.innerHTML;
     return html.indexOf('challenge-platform') !== -1 ||
            html.indexOf('cf_chl_opt') !== -1 ||
+           html.indexOf('cf-error-details') !== -1 ||
+           html.indexOf('cf.errors.css') !== -1 ||
            document.title === 'Just a moment...' ||
+           /Attention Required.*Cloudflare/.test(document.title) ||
            !!document.querySelector('#challenge-running, #challenge-form, .cf-browser-verification');
   })()`);
   return result === true;

package/runtime-src/runtime/setup.ts

@@ -1,11 +1,9 @@
 import { execFileSync } from "node:child_process";
-import { createRequire } from "node:module";
 import { existsSync, mkdirSync, writeFileSync } from "node:fs";
 import os from "node:os";
 import path from "node:path";
 import { ensureDir } from "./paths.js";
-
-const req = createRequire(import.meta.url);
+import { findKuriBinary, getKuriSourceCandidates } from "../kuri/client.js";
 
 export type SetupScope = "auto" | "global" | "project" | "off";
 
@@ -128,18 +126,47 @@ function writeOpenCodeCommand(scope: SetupScope, cwd: string): SetupReport["open
 }
 
 export async function ensureBrowserEngineInstalled(): Promise<SetupReport["browser_engine"]> {
-  try {
-    const { chromium } = await import("playwright-core");
-    if (existsSync(chromium.executablePath())) {
-      return { installed: true, action: "already-installed" };
-    }
+  const binary = findKuriBinary();
+  if (existsSync(binary)) {
+    return { installed: true, action: "already-installed" };
+  }
+
+  const sourceDir = getKuriSourceCandidates().find((candidate) => existsSync(path.join(candidate, "build.zig")));
+  if (!sourceDir) {
+    return {
+      installed: false,
+      action: "failed",
+      message: `Kuri binary not found. Checked ${binary}`,
+    };
+  }
+
+  if (!hasBinary("zig")) {
+    return {
+      installed: false,
+      action: "failed",
+      message: `Kuri source found at ${sourceDir}, but Zig is not installed`,
+    };
+  }
 
-    const agentBrowserBin = req.resolve("agent-browser/bin/agent-browser.js");
-    execFileSync(process.execPath, [agentBrowserBin, "install"], {
+  try {
+    execFileSync("zig", ["build", "-Doptimize=ReleaseFast"], {
+      cwd: sourceDir,
       stdio: "inherit",
       timeout: 300_000,
     });
-    return { installed: true, action: "installed" };
+    const builtBinary = findKuriBinary();
+    if (existsSync(builtBinary)) {
+      return {
+        installed: true,
+        action: "installed",
+        message: `Built Kuri from ${sourceDir}`,
+      };
+    }
+    return {
+      installed: false,
+      action: "failed",
+      message: `Kuri build completed but ${builtBinary} was not created`,
+    };
   } catch (error) {
     const message = error instanceof Error ? error.message : String(error);
     return { installed: false, action: "failed", message };