unbrowse 2.0.3 → 2.0.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "2.0.3",
+  "version": "2.0.5",
   "description": "Reverse-engineer any website into reusable API skills. npm CLI + local engine.",
   "type": "module",
   "bin": {

package/runtime-src/auth/browser-cookies.ts

@@ -112,10 +112,10 @@ function pickFirefoxProfile(profilesRoot: string, profile?: string): string | nu
   return existsSync(candidate) ? candidate : null;
 }
 
-function getFirefoxCookiesPath(profile?: string): string | null {
-  const profilesRoot = getFirefoxProfilesRoot();
-  if (!profilesRoot || !existsSync(profilesRoot)) return null;
-  return pickFirefoxProfile(profilesRoot, profile);
+function getFirefoxCookiesPath(profile?: string, profilesRoot?: string): string | null {
+  const root = profilesRoot ?? getFirefoxProfilesRoot();
+  if (!root || !existsSync(root)) return null;
+  return pickFirefoxProfile(root, profile);
 }
 
 // ---------------------------------------------------------------------------
@@ -335,13 +335,14 @@ export function extractFromChromium(
 
 export function extractFromFirefox(
   domain: string,
-  opts?: { profile?: string },
+  opts?: { profile?: string; profilesRoot?: string },
 ): ExtractionResult {
   const warnings: string[] = [];
-  const dbPath = getFirefoxCookiesPath(opts?.profile);
+  const dbPath = getFirefoxCookiesPath(opts?.profile, opts?.profilesRoot);
+  const browserLabel = opts?.profilesRoot ? "Zen" : "Firefox";
 
   if (!dbPath) {
-    warnings.push("Firefox cookies DB not found");
+    warnings.push(`${browserLabel} cookies DB not found`);
     return { cookies: [], source: null, warnings };
   }
 
@@ -373,14 +374,14 @@ export function extractFromFirefox(
       return results;
     });
 
-    const source = opts?.profile ? `Firefox profile "${opts.profile}"` : "Firefox default profile";
+    const source = opts?.profile ? `${browserLabel} profile "${opts.profile}"` : `${browserLabel} default profile`;
     if (cookies.length === 0) {
       warnings.push(`No cookies for ${domain} found in ${source}`);
     }
     log("auth", `extracted ${cookies.length} cookies for ${domain} from ${source}`);
     return { cookies, source: cookies.length > 0 ? source : null, warnings };
   } catch (err) {
-    warnings.push(`Firefox extraction failed: ${err instanceof Error ? err.message : err}`);
+    warnings.push(`${browserLabel} extraction failed: ${err instanceof Error ? err.message : err}`);
     return { cookies: [], source: null, warnings };
   }
 }
@@ -416,8 +417,62 @@ export function extractBrowserCookies(
     return chromium;
   }
 
-  // Fall back to Chrome
+  // Try Chrome first
   const chrome = extractFromChrome(domain, { profile: opts?.chromeProfile });
-  chrome.warnings.push(...ff.warnings);
-  return chrome;
+  if (chrome.cookies.length > 0) {
+    chrome.warnings.push(...ff.warnings);
+    return chrome;
+  }
+
+  // Auto-discover other Chromium-family browsers
+  const home = homedir();
+  const chromiumBrowsers: Array<{ name: string; userDataDir: string; safeStorageService: string }> =
+    platform() === "darwin"
+      ? [
+          { name: "Arc", userDataDir: join(home, "Library", "Application Support", "Arc", "User Data"), safeStorageService: "Arc Safe Storage" },
+          { name: "Dia", userDataDir: join(home, "Library", "Application Support", "Dia", "User Data"), safeStorageService: "Dia Safe Storage" },
+          { name: "Brave", userDataDir: join(home, "Library", "Application Support", "BraveSoftware", "Brave-Browser"), safeStorageService: "Brave Safe Storage" },
+          { name: "Edge", userDataDir: join(home, "Library", "Application Support", "Microsoft Edge"), safeStorageService: "Microsoft Edge Safe Storage" },
+          { name: "Vivaldi", userDataDir: join(home, "Library", "Application Support", "Vivaldi"), safeStorageService: "Vivaldi Safe Storage" },
+          { name: "Chromium", userDataDir: join(home, "Library", "Application Support", "Chromium"), safeStorageService: "Chromium Safe Storage" },
+        ]
+      : platform() === "linux"
+        ? [
+            { name: "Brave", userDataDir: join(home, ".config", "BraveSoftware", "Brave-Browser"), safeStorageService: "Brave Safe Storage" },
+            { name: "Edge", userDataDir: join(home, ".config", "microsoft-edge"), safeStorageService: "Microsoft Edge Safe Storage" },
+            { name: "Vivaldi", userDataDir: join(home, ".config", "vivaldi"), safeStorageService: "Vivaldi Safe Storage" },
+            { name: "Chromium", userDataDir: join(home, ".config", "chromium"), safeStorageService: "Chromium Safe Storage" },
+          ]
+        : [];
+
+  const allWarnings = [...ff.warnings, ...chrome.warnings];
+  for (const browser of chromiumBrowsers) {
+    if (!existsSync(browser.userDataDir)) continue;
+    const result = extractFromChromium(domain, {
+      userDataDir: browser.userDataDir,
+      browserName: browser.name,
+      safeStorageService: browser.safeStorageService,
+    });
+    if (result.cookies.length > 0) {
+      result.warnings.push(...allWarnings);
+      return result;
+    }
+    allWarnings.push(...result.warnings);
+  }
+
+  // Also try Firefox-based alternatives (Zen)
+  const zenPaths = platform() === "darwin"
+    ? [join(home, "Library", "Application Support", "zen")]
+    : [join(home, ".zen")];
+  for (const zenRoot of zenPaths) {
+    if (!existsSync(zenRoot)) continue;
+    const zenResult = extractFromFirefox(domain, { profilesRoot: zenRoot });
+    if (zenResult.cookies.length > 0) {
+      zenResult.warnings.push(...allWarnings);
+      return zenResult;
+    }
+    allWarnings.push(...zenResult.warnings);
+  }
+
+  return { cookies: [], source: null, warnings: allWarnings };
 }

package/runtime-src/auth/index.ts

@@ -38,95 +38,43 @@ export async function interactiveLogin(
   domain?: string,
 ): Promise<LoginResult> {
   const targetDomain = domain ?? new URL(url).hostname;
-  const profileDir = getProfilePath(targetDomain);
 
   log("auth", `interactiveLogin — url: ${url}, domain: ${targetDomain}`);
 
-  try {
-    fs.mkdirSync(profileDir, { recursive: true });
-
-    // Start Kuri and get a tab
-    await kuri.start();
-    const tabId = await kuri.getDefaultTab();
-    await kuri.networkEnable(tabId);
-
-    // Navigate to login URL
-    await kuri.navigate(tabId, url);
-
-    const startTime = Date.now();
-
-    // Snapshot initial cookies
-    const initialCookies = await kuri.getCookies(tabId);
-    const initialCookieCount = initialCookies.filter((c) => isDomainMatch(c.domain, targetDomain)).length;
-    log("auth", `initial cookies for ${targetDomain}: ${initialCookieCount}`);
-
-    // Wait for user to complete login — detect via cookie changes + URL change
-    let loggedIn = false;
-    let lastLoggedUrl = "";
-    while (Date.now() - startTime < LOGIN_TIMEOUT_MS) {
-      await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
-      const elapsed = Date.now() - startTime;
-
-      try {
-        const currentUrl = await kuri.getCurrentUrl(tabId);
-        const currentDomain = new URL(currentUrl).hostname.toLowerCase();
-        const targetNorm = targetDomain.toLowerCase();
-
-        if (currentUrl !== lastLoggedUrl) {
-          log("auth", `navigated to: ${currentUrl}`);
-          lastLoggedUrl = currentUrl;
-        }
-
-        if (elapsed < MIN_WAIT_MS) continue;
-
-        const isOnTarget = currentDomain === targetNorm || currentDomain.endsWith("." + targetNorm);
-        if (isOnTarget) {
-          const isStillLogin = /\/(login|signin|sign-in|sso|auth|oauth|uas\/login|checkpoint)/.test(new URL(currentUrl).pathname);
+  // Open URL in the user's default browser (visible, not headless)
+  const { exec } = await import("node:child_process");
+  const openCmd = process.platform === "darwin" ? "open" : "xdg-open";
+  exec(`${openCmd} ${JSON.stringify(url)}`);
+  log("auth", `opened ${url} in default browser via ${openCmd}`);
 
-          const currentCookies = await kuri.getCookies(tabId);
-          const currentCookieCount = currentCookies.filter((c) => isDomainMatch(c.domain, targetDomain)).length;
-          const gotNewCookies = currentCookieCount > initialCookieCount;
+  // Poll extractBrowserAuth until cookies appear or timeout
+  const startTime = Date.now();
+  while (Date.now() - startTime < LOGIN_TIMEOUT_MS) {
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
 
-          if (!isStillLogin && gotNewCookies) {
-            loggedIn = true;
-            log("auth", `login complete — ${currentUrl} (cookies: ${initialCookieCount} → ${currentCookieCount})`);
-            break;
-          }
-
-          if (!isStillLogin && currentCookieCount > 0) {
-            loggedIn = true;
-            log("auth", `already logged in — ${currentUrl} (${currentCookieCount} cookies present)`);
-            break;
-          }
-        }
-      } catch { /* page navigating */ }
-    }
-
-    if (!loggedIn) {
-      log("auth", `login wait ended after ${Math.round((Date.now() - startTime) / 1000)}s — capturing cookies anyway`);
+    try {
+      const result = await extractBrowserAuth(targetDomain);
+      if (result.success && result.cookies_stored > 0) {
+        log("auth", `login detected — ${result.cookies_stored} cookies captured for ${targetDomain}`);
+        return result;
+      }
+    } catch (err) {
+      log("auth", `poll error: ${err instanceof Error ? err.message : err}`);
     }
 
-    // Extract and store cookies
-    const cookies = await kuri.getCookies(tabId);
-    const domainCookies = cookies.filter((c) => isDomainMatch(c.domain, targetDomain));
-
-    if (domainCookies.length === 0) {
-      return { success: false, domain: targetDomain, cookies_stored: 0, error: "No cookies captured for domain" };
+    // Log progress every 10s
+    const elapsed = Date.now() - startTime;
+    if (elapsed % 10_000 < POLL_INTERVAL_MS) {
+      log("auth", `waiting for login... ${Math.round(elapsed / 1000)}s elapsed`);
     }
-
-    const storableCookies = domainCookies.map((c) => ({
-      name: c.name, value: c.value, domain: c.domain, path: c.path,
-      secure: c.secure, httpOnly: c.httpOnly, sameSite: c.sameSite, expires: c.expires,
-    }));
-
-    const vaultKey = `auth:${getRegistrableDomain(targetDomain)}`;
-    await storeCredential(vaultKey, JSON.stringify({ cookies: storableCookies }));
-    log("auth", `stored ${storableCookies.length} cookies under ${vaultKey}`);
-
-    return { success: true, domain: targetDomain, cookies_stored: storableCookies.length };
-  } finally {
-    // Cleanup handled by Kuri's tab management
   }
+
+  return {
+    success: false,
+    domain: targetDomain,
+    cookies_stored: 0,
+    error: `Login timed out after ${LOGIN_TIMEOUT_MS / 1000}s — no cookies detected in browser`,
+  };
 }
 
 /**

package/runtime-src/capture/index.ts

@@ -6,6 +6,40 @@ import { log } from "../logger.js";
 // BUG-GC-012: Use a real Chrome UA — HeadlessChrome is actively blocked by Google and others.
 const CHROME_UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36";
 
+// Stealth script — hides headless Chrome indicators from bot detection.
+// Ported from kuri's cdp/js/stealth.js (commit 4dbbd89).
+const STEALTH_SCRIPT = `
+Object.defineProperty(navigator, 'webdriver', { get: () => false, configurable: true });
+Object.defineProperty(navigator, 'plugins', {
+  get: () => {
+    const p = [
+      { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
+      { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
+      { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
+    ];
+    p.length = 3;
+    return p;
+  },
+  configurable: true,
+});
+Object.defineProperty(navigator, 'languages', { get: () => ['en-US', 'en'], configurable: true });
+if (!window.chrome) window.chrome = {};
+if (!window.chrome.runtime) window.chrome.runtime = { connect: () => {}, sendMessage: () => {}, id: undefined };
+const origQuery = window.navigator.permissions?.query;
+if (origQuery) {
+  window.navigator.permissions.query = (p) =>
+    p.name === 'notifications' ? Promise.resolve({ state: Notification.permission }) : origQuery(p);
+}
+try {
+  const d = Object.getOwnPropertyDescriptor(HTMLIFrameElement.prototype, 'contentWindow');
+  if (d) Object.defineProperty(HTMLIFrameElement.prototype, 'contentWindow', { get: function() { return d.get.call(this); } });
+} catch {}
+Object.defineProperty(navigator, 'userAgent', {
+  get: () => '${CHROME_UA}',
+  configurable: true,
+});
+`;
+
 // Tab semaphore: max 3 concurrent capture tabs
 const MAX_CONCURRENT_TABS = 3;
 let activeTabs = 0;
@@ -654,6 +688,11 @@ export async function captureSession(
       await injectCookies(tabId, cookies);
     }
 
+    // Inject stealth patches — hide headless Chrome indicators from bot detection
+    try {
+      await kuri.evaluate(tabId, STEALTH_SCRIPT);
+    } catch { /* best-effort */ }
+
     // Start HAR recording
     await kuri.harStart(tabId);
 
@@ -662,16 +701,23 @@ export async function captureSession(
     try { pageDomain = getRegistrableDomain(new URL(url).hostname); } catch { /* bad url */ }
 
     // Inject fetch/XHR interceptor BEFORE navigation to capture all response bodies
-    // Navigate directly to target URL — skip origin pre-navigation to save 1-2s on heavy SPAs.
-    // The interceptor is re-injected after navigation anyway (page context resets on navigate).
-    await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT).catch(() => {});
+    // Navigate to origin first so cookies are applied in the correct domain context
+    // before the full page load — required for sites like LinkedIn that check auth on first load.
+    try {
+      const origin = new URL(url).origin;
+      await kuri.navigate(tabId, origin);
+      await new Promise((r) => setTimeout(r, 500));
+      await kuri.evaluate(tabId, STEALTH_SCRIPT);
+      await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT);
+    } catch { /* best-effort */ }
 
     // Navigate to target URL
     await kuri.navigate(tabId, url);
 
-    // Re-inject interceptor after navigation (page context resets on navigate)
+    // Re-inject stealth + interceptor after navigation (page context resets on navigate)
     try {
       await new Promise((r) => setTimeout(r, 300));
+      await kuri.evaluate(tabId, STEALTH_SCRIPT);
       await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT);
     } catch { /* page may not be ready */ }
 

package/runtime-src/execution/index.ts

@@ -1792,13 +1792,15 @@ export async function executeEndpoint(
 
       // CSRF token auto-detection (bird pattern): many sites require CSRF tokens
       // as both a cookie AND a header. Detect common patterns and replay them.
-      if (!headers["x-csrf-token"] && !headers["x-xsrf-token"]) {
+      if (!headers["x-csrf-token"] && !headers["x-xsrf-token"] && !headers["csrf-token"]) {
         const csrfCookie = cookies.find((c) =>
-          /^(ct0|csrf_token|_csrf|csrftoken|XSRF-TOKEN|_xsrf)$/i.test(c.name)
+          /^(ct0|csrf_token|_csrf|csrftoken|XSRF-TOKEN|_xsrf|JSESSIONID)$/i.test(c.name)
         );
         if (csrfCookie) {
           const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
-          headers["x-csrf-token"] = v;
+          // LinkedIn uses "csrf-token" header derived from JSESSIONID
+          const headerName = csrfCookie.name === "JSESSIONID" ? "csrf-token" : "x-csrf-token";
+          headers[headerName] = v;
         }
       }
     }

package/runtime-src/reverse-engineer/index.ts

@@ -381,8 +381,30 @@ function inferCsrfPlan(req: RawRequest, parsedBody?: unknown): CsrfPlan | undefi
     Object.entries(req.request_headers).map(([key, value]) => [key.toLowerCase(), value]),
   );
   const cookies = parseCookieHeader(headers["cookie"]);
-  const csrfCookieNames = Object.keys(cookies).filter((name) => /^(ct0|csrf_token|_csrf|csrftoken|xsrf-token|_xsrf)$/i.test(name));
-  const headerName = ["x-csrf-token", "x-xsrf-token", "x-csrftoken"].find((name) => typeof headers[name] === "string" && headers[name].length > 0);
+  const csrfCookieNames = Object.keys(cookies).filter((name) => /^(ct0|csrf_token|_csrf|csrftoken|xsrf-token|_xsrf|JSESSIONID)$/i.test(name));
+  const headerName = ["x-csrf-token", "x-xsrf-token", "x-csrftoken", "csrf-token"].find((name) => typeof headers[name] === "string" && headers[name].length > 0);
+
+  // Also detect CSRF by value matching: if any cookie value appears as a header value,
+  // that's a CSRF token pattern regardless of naming convention
+  if (!headerName && csrfCookieNames.length === 0) {
+    for (const [cookieName, cookieValue] of Object.entries(cookies)) {
+      if (!cookieValue || cookieValue.length < 8) continue;
+      const unquoted = cookieValue.startsWith('"') && cookieValue.endsWith('"') ? cookieValue.slice(1, -1) : cookieValue;
+      for (const [hName, hValue] of Object.entries(headers)) {
+        if (hName === "cookie" || hName === "host" || hName === "content-length") continue;
+        const hUnquoted = hValue.startsWith('"') && hValue.endsWith('"') ? hValue.slice(1, -1) : hValue;
+        if (unquoted === hUnquoted && unquoted.length >= 8) {
+          return {
+            source: "cookie",
+            param_name: hName,
+            refresh_on_401: true,
+            extractor_sequence: [cookieName],
+          };
+        }
+      }
+    }
+  }
+
   if (headerName && csrfCookieNames.length > 0) {
     return {
       source: "cookie",