unbrowse 2.0.1 → 2.0.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "2.0.1",
+  "version": "2.0.2",
   "description": "Reverse-engineer any website into reusable API skills. npm CLI + local engine.",
   "type": "module",
   "bin": {

package/runtime-src/capture/index.ts

@@ -137,7 +137,9 @@ export function isBlockedAppShell(html?: string): boolean {
     /switch to a supported browser/i.test(html) ||
     /Something went wrong, but don.?t fret/i.test(html) ||
     /class=["']errorContainer["']/i.test(html) ||
-    /#placeholder,\s*#react-root\s*\{\s*display:\s*none/i.test(html)
+    /#placeholder,\s*#react-root\s*\{\s*display:\s*none/i.test(html) ||
+    /Attention Required!\s*\|\s*Cloudflare/i.test(html) ||
+    /cf-error-details|cf\.errors\.css/i.test(html)
   );
 }
 
@@ -436,7 +438,15 @@ async function waitForContentReady(
   responseBodies?: Map<string, string>,
 ): Promise<void> {
   // Phase 1: Initial settle — let the page start rendering
-  await new Promise((r) => setTimeout(r, 2000));
+  await new Promise((r) => setTimeout(r, 1000));
+
+  // Early exit: if interceptor already captured API responses, page is loaded enough
+  if (responseBodies && responseBodies.size > 0) {
+    log("capture", `early exit: ${responseBodies.size} API responses already captured during navigation`);
+    // Brief extra settle to catch any trailing responses
+    await new Promise((r) => setTimeout(r, 500));
+    return;
+  }
 
   // Phase 2: Cloudflare challenge detection and wait
   try {
@@ -453,7 +463,21 @@ async function waitForContentReady(
   }
 
   // Phase 3: Wait for document ready state (replaces networkidle)
-  await waitForReadyState(tabId, 8000);
+  await waitForReadyState(tabId, 5000);
+
+  // Early exit: check again after readyState — SPAs often fire API calls during hydration
+  if (responseBodies) {
+    const intercepted = await collectInterceptedRequests(tabId);
+    for (const entry of intercepted) {
+      if (entry.response_body && !entry.is_js) {
+        responseBodies.set(entry.url, entry.response_body);
+      }
+    }
+    if (responseBodies.size > 0) {
+      log("capture", `early exit after readyState: ${responseBodies.size} API responses captured`);
+      return;
+    }
+  }
 
   // Phase 4: Intent-aware API wait — poll intercepted requests for matching API URLs
   if (captureUrl && responseBodies) {
@@ -464,8 +488,8 @@ async function waitForContentReady(
     if (wantedHints.length > 0) {
       log("capture", `intent-aware wait: looking for API matching one of [${wantedHints.join(", ")}] (from ${captureUrl})`);
       const intentStart = Date.now();
-      const INTENT_MAX_WAIT = 15000;
-      const INTENT_POLL_INTERVAL = 1500;
+      const INTENT_MAX_WAIT = 8000;
+      const INTENT_POLL_INTERVAL = 1000;
       while (Date.now() - intentStart < INTENT_MAX_WAIT) {
         await new Promise((r) => setTimeout(r, INTENT_POLL_INTERVAL));
         // Check newly intercepted requests
@@ -505,7 +529,7 @@ async function waitForContentReady(
       await new Promise((r) => setTimeout(r, 1200));
       await kuri.evaluate(tabId, "window.scrollTo(0, 0)");
       if (responseBodies.size === before) {
-        await new Promise((r) => setTimeout(r, 2000));
+        await new Promise((r) => setTimeout(r, 1500));
       }
     } catch {
       // non-fatal
@@ -638,14 +662,9 @@ export async function captureSession(
     try { pageDomain = getRegistrableDomain(new URL(url).hostname); } catch { /* bad url */ }
 
     // Inject fetch/XHR interceptor BEFORE navigation to capture all response bodies
-    // Navigate to origin first so the interceptor runs in the correct context
-    try {
-      const origin = new URL(url).origin;
-      await kuri.navigate(tabId, origin);
-      await new Promise((r) => setTimeout(r, 500));
-    } catch { /* best-effort */ }
-
-    await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT);
+    // Navigate directly to target URL — skip origin pre-navigation to save 1-2s on heavy SPAs.
+    // The interceptor is re-injected after navigation anyway (page context resets on navigate).
+    await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT).catch(() => {});
 
     // Navigate to target URL
     await kuri.navigate(tabId, url);
@@ -707,10 +726,14 @@ export async function captureSession(
       log("capture", `response body captured: ${bodyUrl.substring(0, 150)}`);
     }
 
+
     let final_url = url;
     let html: string | undefined;
     try {
-      final_url = await kuri.getCurrentUrl(tabId);
+      const rawUrl = await kuri.getCurrentUrl(tabId);
+      final_url = typeof rawUrl === "string" ? rawUrl : String(rawUrl ?? url);
+      // Validate it's actually a URL, fall back to original if not
+      try { new URL(final_url); } catch { final_url = url; }
       html = await kuri.getPageHtml(tabId);
     } catch {}
 
@@ -779,6 +802,14 @@ export async function captureSession(
       responseBodyCount < 10 &&
       !hasUsefulCapturedResponses(responseBodies.keys(), url, intent)
     ) {
+      // On ephemeral retry, if still blocked by Cloudflare WAF, throw auth_required
+      // so the caller can surface a login prompt instead of retrying forever
+      if (options?.forceEphemeral && html && /Cloudflare|cf\.errors\.css|cf-error-details/i.test(html)) {
+        throw Object.assign(new Error("cloudflare_waf_block"), {
+          code: "auth_required",
+          login_url: url,
+        });
+      }
       retryFreshTab = true;
       log("capture", `rendered blocked app shell for ${url}; retrying with fresh tab`);
     } else {
@@ -807,7 +838,7 @@ export async function captureSession(
     await resetTab(tabId);
     releaseTabSlot(tabId);
   }
-  if (retryFreshTab) {
+  if (retryFreshTab && !options?.forceEphemeral) {
     return captureSession(url, authHeaders, cookies, intent, { forceEphemeral: true });
   }
   if (captureError) throw captureError;

package/runtime-src/execution/index.ts

@@ -939,7 +939,7 @@ async function executeBrowserCapture(
     skill.endpoints.find((endpoint) => typeof endpoint.trigger_url === "string" && endpoint.trigger_url)?.trigger_url ||
     skill.endpoints.find((endpoint) => !/\{[^}]+\}/.test(endpoint.url_template))?.url_template ||
     "";
-  const url = String(params.url ?? fallbackUrl);
+  const url = typeof params.url === "string" ? params.url : String(params.url ?? fallbackUrl);
   const intent = String(params.intent ?? skill.intent_signature);
   if (!url) throw new Error("browser-capture skill requires params.url");
 
@@ -981,7 +981,33 @@ async function executeBrowserCapture(
     usedStoredAuth,
   );
   if (documentSeed) return documentSeed;
-  const captured = await captureSession(url, authHeaders, cookies, intent);
+  let captured;
+  try {
+    captured = await captureSession(url, authHeaders, cookies, intent);
+  } catch (captureErr: unknown) {
+    const err = captureErr as Error & { code?: string; login_url?: string };
+    if (err.code === "auth_required") {
+      const trace: ExecutionTrace = stampTrace({
+        trace_id: traceId,
+        skill_id: skill.skill_id,
+        endpoint_id: "browser-capture",
+        started_at: startedAt,
+        completed_at: new Date().toISOString(),
+        success: false,
+        error: "auth_required",
+      });
+      return {
+        trace,
+        result: {
+          error: "auth_required",
+          provider: "cloudflare",
+          login_url: err.login_url ?? url,
+          message: `Site is blocked by Cloudflare WAF. Run: unbrowse login --url "${url}" to authenticate interactively.`,
+        },
+      };
+    }
+    throw captureErr;
+  }
 
   const finalDomain = (() => {
     try { return new URL(captured.final_url).hostname; } catch { return targetDomain; }
@@ -990,7 +1016,7 @@ async function executeBrowserCapture(
   const LOGIN_PATHS = /\/(login|signin|sign-in|sso|auth|uas\/login|checkpoint|oauth)/i;
 
   const redirectedToAuth = finalDomain !== targetDomain && AUTH_PROVIDERS.test(finalDomain);
-  const redirectedToLogin = captured.final_url !== url && LOGIN_PATHS.test(new URL(captured.final_url).pathname);
+  const redirectedToLogin = captured.final_url !== url && (() => { try { return LOGIN_PATHS.test(new URL(String(captured.final_url)).pathname); } catch { return false; } })();
 
   if (redirectedToAuth || redirectedToLogin) {
     const trace: ExecutionTrace = stampTrace({

package/runtime-src/extraction/index.ts

@@ -1278,18 +1278,38 @@ export function extractFromDOMWithHint(
  * the best match for the given intent.
  */
 export function extractFromDOM(html: string, intent: string): ExtractionResult {
+  // Cap HTML size to prevent cheerio from hanging on massive pages
+  const MAX_HTML_SIZE = 300_000;
+  let workingHtml = html;
+  if (workingHtml.length > MAX_HTML_SIZE) {
+    // Strip attribute bloat first (class/style/data-* attributes inflate HTML 2-3x)
+    workingHtml = workingHtml
+      .replace(/\s+class="[^"]*"/g, "")
+      .replace(/\s+style="[^"]*"/g, "")
+      .replace(/\s+data-[a-z][-a-z]*="[^"]*"/g, "");
+    // If still too large, truncate keeping body content
+    if (workingHtml.length > MAX_HTML_SIZE) {
+      const bodyStart = workingHtml.indexOf("<body");
+      if (bodyStart > 0) {
+        workingHtml = workingHtml.substring(0, Math.max(MAX_HTML_SIZE, bodyStart + MAX_HTML_SIZE));
+      } else {
+        workingHtml = workingHtml.substring(0, MAX_HTML_SIZE);
+      }
+    }
+  }
+
   // Extract SPA-embedded data from raw HTML BEFORE cleanDOM strips scripts
-  const spaStructures = extractSPAData(html);
-  const flashStructures = extractFlashNoticeSpecial(html, intent);
-  const cleaned = cleanDOM(html);
-  const githubStructures = extractGitHubSpecial(html, intent);
-  const linkedInStructures = extractLinkedInSpecial(html, intent);
-  const packageSearchStructures = extractPackageSearchSpecial(html, intent);
-  const xProfileStructures = extractXProfileSpecial(html, intent);
-  const postStructures = extractPostSpecial(html, intent);
-  const trendStructures = extractTrendSpecial(html, intent);
-  const definitionStructures = extractDefinitionSpecial(html, intent);
-  const courseStructures = extractCourseSearchSpecial(html, intent);
+  const spaStructures = extractSPAData(workingHtml);
+  const flashStructures = extractFlashNoticeSpecial(workingHtml, intent);
+  const cleaned = cleanDOM(workingHtml);
+  const githubStructures = extractGitHubSpecial(workingHtml, intent);
+  const linkedInStructures = extractLinkedInSpecial(workingHtml, intent);
+  const packageSearchStructures = extractPackageSearchSpecial(workingHtml, intent);
+  const xProfileStructures = extractXProfileSpecial(workingHtml, intent);
+  const postStructures = extractPostSpecial(workingHtml, intent);
+  const trendStructures = extractTrendSpecial(workingHtml, intent);
+  const definitionStructures = extractDefinitionSpecial(workingHtml, intent);
+  const courseStructures = extractCourseSearchSpecial(workingHtml, intent);
   const structures = [...flashStructures, ...githubStructures, ...linkedInStructures, ...packageSearchStructures, ...xProfileStructures, ...postStructures, ...trendStructures, ...definitionStructures, ...courseStructures, ...spaStructures, ...parseStructured(cleaned)]
     .map((structure) => normalizeStructureForIntent(structure, intent));
 
@@ -1306,7 +1326,17 @@ export function extractFromDOM(html: string, intent: string): ExtractionResult {
 
   scored.sort((a, b) => b.score - a.score);
 
-  const bestPassing = scored.find((candidate) => assessIntentResult(candidate.structure.data, intent).verdict === "pass");
+  const passing = scored.filter((candidate) => assessIntentResult(candidate.structure.data, intent).verdict === "pass");
+  const bestPassing = (() => {
+    if (passing.length === 0) return undefined;
+    const bestPassingOverall = passing[0];
+    const bestPassingSpa = passing.find((candidate) => candidate.structure.type.startsWith("spa-"));
+    // Prefer cleaner SPA payloads when they're effectively tied with DOM-derived candidates.
+    if (bestPassingSpa && bestPassingOverall && bestPassingSpa.score >= bestPassingOverall.score - 2) {
+      return bestPassingSpa;
+    }
+    return bestPassingOverall;
+  })();
   if (bestPassing) {
     return {
       data: bestPassing.structure.data,
@@ -1325,7 +1355,17 @@ export function extractFromDOM(html: string, intent: string): ExtractionResult {
       selector: best.structure.selector,
     };
   }
-  const hasClearWinner = scored.length === 1 || best.score > scored[1].score * 1.5;
+
+  if (scored.length === 1) {
+    return {
+      data: best.structure.data,
+      extraction_method: best.structure.type,
+      confidence: computeConfidence(best.structure, best.score),
+      selector: best.structure.selector,
+    };
+  }
+
+  const hasClearWinner = best.score > scored[1].score * 1.5;
 
   if (hasClearWinner && best.score > 0) {
     return {

package/runtime-src/kuri/client.ts

@@ -339,12 +339,16 @@ export async function getDefaultTab(): Promise<string> {
   throw new Error("No tabs available and failed to create one");
 }
 
+/** Trigger Kuri's /discover to sync Chrome tabs into Kuri's registry. */
 /** Trigger Kuri's /discover to sync Chrome tabs into Kuri's registry. */
 async function ensureTabsDiscovered(): Promise<void> {
   try {
-    await kuriGet("/discover");
+    // Pass CDP URL as query param so /discover works even if Kuri was started without CDP_URL env
+    const params: Record<string, string> = {};
+    if (kuriCdpPort) params.cdp_url = `ws://127.0.0.1:${kuriCdpPort}`;
+    await kuriGet("/discover", params);
   } catch {
-    // /discover may fail if CDP_URL not set — that's handled by start()
+    // /discover may fail if no Chrome running — that's OK
   }
 }
 
@@ -353,12 +357,35 @@ export async function navigate(tabId: string, url: string): Promise<void> {
   await kuriGet("/navigate", { tab_id: tabId, url });
 }
 
+/** Evaluate JavaScript in tab context. */
+/** Evaluate JavaScript in tab context. */
+/** Evaluate JavaScript in tab context. */
 /** Evaluate JavaScript in tab context. */
 export async function evaluate(tabId: string, expression: string): Promise<unknown> {
-  const raw = (await kuriGet("/evaluate", { tab_id: tabId, expression })) as {
+  let raw: {
     id?: number;
     result?: { result?: { type?: string; value?: unknown; description?: string }; exceptionDetails?: unknown };
   };
+  if (expression.length > 2000) {
+    // Use POST with raw text body for large expressions to avoid URL length limits
+    const url = kuriUrl("/evaluate", { tab_id: tabId });
+    const controller = new AbortController();
+    const timeout = setTimeout(() => controller.abort(), KURI_REQUEST_TIMEOUT_MS);
+    try {
+      const res = await fetch(url, {
+        method: "POST",
+        headers: { "Content-Type": "text/plain" },
+        body: expression,
+        signal: controller.signal,
+      });
+      const text = await res.text();
+      try { raw = JSON.parse(text); } catch { raw = text as never; }
+    } finally {
+      clearTimeout(timeout);
+    }
+  } else {
+    raw = (await kuriGet("/evaluate", { tab_id: tabId, expression })) as typeof raw;
+  }
   // CDP Runtime.evaluate response: { id, result: { result: { type, value } } }
   const inner = raw?.result?.result;
   if (!inner) return raw;
@@ -483,7 +510,10 @@ export async function hasCloudflareChallenge(tabId: string): Promise<boolean> {
     var html = document.documentElement.innerHTML;
     return html.indexOf('challenge-platform') !== -1 ||
            html.indexOf('cf_chl_opt') !== -1 ||
+           html.indexOf('cf-error-details') !== -1 ||
+           html.indexOf('cf.errors.css') !== -1 ||
            document.title === 'Just a moment...' ||
+           /Attention Required.*Cloudflare/.test(document.title) ||
            !!document.querySelector('#challenge-running, #challenge-form, .cf-browser-verification');
   })()`);
   return result === true;