unbrowse 2.0.2 → 2.0.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "2.0.2",
+  "version": "2.0.4",
   "description": "Reverse-engineer any website into reusable API skills. npm CLI + local engine.",
   "type": "module",
   "bin": {

package/runtime-src/auth/browser-cookies.ts

@@ -112,10 +112,10 @@ function pickFirefoxProfile(profilesRoot: string, profile?: string): string | nu
   return existsSync(candidate) ? candidate : null;
 }
 
-function getFirefoxCookiesPath(profile?: string): string | null {
-  const profilesRoot = getFirefoxProfilesRoot();
-  if (!profilesRoot || !existsSync(profilesRoot)) return null;
-  return pickFirefoxProfile(profilesRoot, profile);
+function getFirefoxCookiesPath(profile?: string, profilesRoot?: string): string | null {
+  const root = profilesRoot ?? getFirefoxProfilesRoot();
+  if (!root || !existsSync(root)) return null;
+  return pickFirefoxProfile(root, profile);
 }
 
 // ---------------------------------------------------------------------------
@@ -136,9 +136,10 @@ function getChromiumDecryptionKey(opts?: ChromiumCookieSourceOptions): Buffer |
   if (platform() !== "darwin") return null; // TODO: Linux/Windows support
 
   try {
-    const keyOutput = execSync(
-      `security find-generic-password -s "${service.replace(/"/g, '\\"')}" -w 2>/dev/null || echo ""`,
-      { encoding: "utf8" },
+    const keyOutput = execFileSync(
+      "security",
+      ["find-generic-password", "-s", service, "-w"],
+      { encoding: "utf8", stdio: ["pipe", "pipe", "pipe"] },
     ).trim();
     if (!keyOutput) return null;
 
@@ -241,9 +242,13 @@ function buildDomainWhereClause(domain: string, column: string): string {
     `www.${reg}`,
     `.www.${reg}`,
   ]);
-  const escaped = [...variants].map((d) => `'${d.replace(/'/g, "''")}'`);
-  // Also match any subdomain via LIKE (e.g. .api.example.com, .sg.example.com)
-  const likePattern = `'%.${reg.replace(/'/g, "''")}'`;
+  // Use parameterized-safe quoting: reject any domain containing single quotes
+  for (const d of variants) {
+    if (d.includes("'")) throw new Error(`Invalid domain for cookie query: ${d}`);
+  }
+  const escaped = [...variants].map((d) => `'${d}'`);
+  const likeReg = reg.includes("'") ? reg : reg;
+  const likePattern = `'%.${likeReg}'`;
   return `(${column} IN (${escaped.join(", ")}) OR ${column} LIKE ${likePattern})`;
 }
 
@@ -330,13 +335,14 @@ export function extractFromChromium(
 
 export function extractFromFirefox(
   domain: string,
-  opts?: { profile?: string },
+  opts?: { profile?: string; profilesRoot?: string },
 ): ExtractionResult {
   const warnings: string[] = [];
-  const dbPath = getFirefoxCookiesPath(opts?.profile);
+  const dbPath = getFirefoxCookiesPath(opts?.profile, opts?.profilesRoot);
+  const browserLabel = opts?.profilesRoot ? "Zen" : "Firefox";
 
   if (!dbPath) {
-    warnings.push("Firefox cookies DB not found");
+    warnings.push(`${browserLabel} cookies DB not found`);
     return { cookies: [], source: null, warnings };
   }
 
@@ -368,14 +374,14 @@ export function extractFromFirefox(
       return results;
     });
 
-    const source = opts?.profile ? `Firefox profile "${opts.profile}"` : "Firefox default profile";
+    const source = opts?.profile ? `${browserLabel} profile "${opts.profile}"` : `${browserLabel} default profile`;
     if (cookies.length === 0) {
       warnings.push(`No cookies for ${domain} found in ${source}`);
     }
     log("auth", `extracted ${cookies.length} cookies for ${domain} from ${source}`);
     return { cookies, source: cookies.length > 0 ? source : null, warnings };
   } catch (err) {
-    warnings.push(`Firefox extraction failed: ${err instanceof Error ? err.message : err}`);
+    warnings.push(`${browserLabel} extraction failed: ${err instanceof Error ? err.message : err}`);
     return { cookies: [], source: null, warnings };
   }
 }
@@ -411,8 +417,62 @@ export function extractBrowserCookies(
     return chromium;
   }
 
-  // Fall back to Chrome
+  // Try Chrome first
   const chrome = extractFromChrome(domain, { profile: opts?.chromeProfile });
-  chrome.warnings.push(...ff.warnings);
-  return chrome;
+  if (chrome.cookies.length > 0) {
+    chrome.warnings.push(...ff.warnings);
+    return chrome;
+  }
+
+  // Auto-discover other Chromium-family browsers
+  const home = homedir();
+  const chromiumBrowsers: Array<{ name: string; userDataDir: string; safeStorageService: string }> =
+    platform() === "darwin"
+      ? [
+          { name: "Arc", userDataDir: join(home, "Library", "Application Support", "Arc", "User Data"), safeStorageService: "Arc Safe Storage" },
+          { name: "Dia", userDataDir: join(home, "Library", "Application Support", "Dia", "User Data"), safeStorageService: "Dia Safe Storage" },
+          { name: "Brave", userDataDir: join(home, "Library", "Application Support", "BraveSoftware", "Brave-Browser"), safeStorageService: "Brave Safe Storage" },
+          { name: "Edge", userDataDir: join(home, "Library", "Application Support", "Microsoft Edge"), safeStorageService: "Microsoft Edge Safe Storage" },
+          { name: "Vivaldi", userDataDir: join(home, "Library", "Application Support", "Vivaldi"), safeStorageService: "Vivaldi Safe Storage" },
+          { name: "Chromium", userDataDir: join(home, "Library", "Application Support", "Chromium"), safeStorageService: "Chromium Safe Storage" },
+        ]
+      : platform() === "linux"
+        ? [
+            { name: "Brave", userDataDir: join(home, ".config", "BraveSoftware", "Brave-Browser"), safeStorageService: "Brave Safe Storage" },
+            { name: "Edge", userDataDir: join(home, ".config", "microsoft-edge"), safeStorageService: "Microsoft Edge Safe Storage" },
+            { name: "Vivaldi", userDataDir: join(home, ".config", "vivaldi"), safeStorageService: "Vivaldi Safe Storage" },
+            { name: "Chromium", userDataDir: join(home, ".config", "chromium"), safeStorageService: "Chromium Safe Storage" },
+          ]
+        : [];
+
+  const allWarnings = [...ff.warnings, ...chrome.warnings];
+  for (const browser of chromiumBrowsers) {
+    if (!existsSync(browser.userDataDir)) continue;
+    const result = extractFromChromium(domain, {
+      userDataDir: browser.userDataDir,
+      browserName: browser.name,
+      safeStorageService: browser.safeStorageService,
+    });
+    if (result.cookies.length > 0) {
+      result.warnings.push(...allWarnings);
+      return result;
+    }
+    allWarnings.push(...result.warnings);
+  }
+
+  // Also try Firefox-based alternatives (Zen)
+  const zenPaths = platform() === "darwin"
+    ? [join(home, "Library", "Application Support", "zen")]
+    : [join(home, ".zen")];
+  for (const zenRoot of zenPaths) {
+    if (!existsSync(zenRoot)) continue;
+    const zenResult = extractFromFirefox(domain, { profilesRoot: zenRoot });
+    if (zenResult.cookies.length > 0) {
+      zenResult.warnings.push(...allWarnings);
+      return zenResult;
+    }
+    allWarnings.push(...zenResult.warnings);
+  }
+
+  return { cookies: [], source: null, warnings: allWarnings };
 }

package/runtime-src/auth/index.ts

@@ -38,95 +38,43 @@ export async function interactiveLogin(
   domain?: string,
 ): Promise<LoginResult> {
   const targetDomain = domain ?? new URL(url).hostname;
-  const profileDir = getProfilePath(targetDomain);
 
   log("auth", `interactiveLogin — url: ${url}, domain: ${targetDomain}`);
 
-  try {
-    fs.mkdirSync(profileDir, { recursive: true });
-
-    // Start Kuri and get a tab
-    await kuri.start();
-    const tabId = await kuri.getDefaultTab();
-    await kuri.networkEnable(tabId);
-
-    // Navigate to login URL
-    await kuri.navigate(tabId, url);
-
-    const startTime = Date.now();
-
-    // Snapshot initial cookies
-    const initialCookies = await kuri.getCookies(tabId);
-    const initialCookieCount = initialCookies.filter((c) => isDomainMatch(c.domain, targetDomain)).length;
-    log("auth", `initial cookies for ${targetDomain}: ${initialCookieCount}`);
-
-    // Wait for user to complete login — detect via cookie changes + URL change
-    let loggedIn = false;
-    let lastLoggedUrl = "";
-    while (Date.now() - startTime < LOGIN_TIMEOUT_MS) {
-      await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
-      const elapsed = Date.now() - startTime;
-
-      try {
-        const currentUrl = await kuri.getCurrentUrl(tabId);
-        const currentDomain = new URL(currentUrl).hostname.toLowerCase();
-        const targetNorm = targetDomain.toLowerCase();
-
-        if (currentUrl !== lastLoggedUrl) {
-          log("auth", `navigated to: ${currentUrl}`);
-          lastLoggedUrl = currentUrl;
-        }
-
-        if (elapsed < MIN_WAIT_MS) continue;
-
-        const isOnTarget = currentDomain === targetNorm || currentDomain.endsWith("." + targetNorm);
-        if (isOnTarget) {
-          const isStillLogin = /\/(login|signin|sign-in|sso|auth|oauth|uas\/login|checkpoint)/.test(new URL(currentUrl).pathname);
+  // Open URL in the user's default browser (visible, not headless)
+  const { exec } = await import("node:child_process");
+  const openCmd = process.platform === "darwin" ? "open" : "xdg-open";
+  exec(`${openCmd} ${JSON.stringify(url)}`);
+  log("auth", `opened ${url} in default browser via ${openCmd}`);
 
-          const currentCookies = await kuri.getCookies(tabId);
-          const currentCookieCount = currentCookies.filter((c) => isDomainMatch(c.domain, targetDomain)).length;
-          const gotNewCookies = currentCookieCount > initialCookieCount;
+  // Poll extractBrowserAuth until cookies appear or timeout
+  const startTime = Date.now();
+  while (Date.now() - startTime < LOGIN_TIMEOUT_MS) {
+    await new Promise((r) => setTimeout(r, POLL_INTERVAL_MS));
 
-          if (!isStillLogin && gotNewCookies) {
-            loggedIn = true;
-            log("auth", `login complete — ${currentUrl} (cookies: ${initialCookieCount} → ${currentCookieCount})`);
-            break;
-          }
-
-          if (!isStillLogin && currentCookieCount > 0) {
-            loggedIn = true;
-            log("auth", `already logged in — ${currentUrl} (${currentCookieCount} cookies present)`);
-            break;
-          }
-        }
-      } catch { /* page navigating */ }
-    }
-
-    if (!loggedIn) {
-      log("auth", `login wait ended after ${Math.round((Date.now() - startTime) / 1000)}s — capturing cookies anyway`);
+    try {
+      const result = await extractBrowserAuth(targetDomain);
+      if (result.success && result.cookies_stored > 0) {
+        log("auth", `login detected — ${result.cookies_stored} cookies captured for ${targetDomain}`);
+        return result;
+      }
+    } catch (err) {
+      log("auth", `poll error: ${err instanceof Error ? err.message : err}`);
     }
 
-    // Extract and store cookies
-    const cookies = await kuri.getCookies(tabId);
-    const domainCookies = cookies.filter((c) => isDomainMatch(c.domain, targetDomain));
-
-    if (domainCookies.length === 0) {
-      return { success: false, domain: targetDomain, cookies_stored: 0, error: "No cookies captured for domain" };
+    // Log progress every 10s
+    const elapsed = Date.now() - startTime;
+    if (elapsed % 10_000 < POLL_INTERVAL_MS) {
+      log("auth", `waiting for login... ${Math.round(elapsed / 1000)}s elapsed`);
     }
-
-    const storableCookies = domainCookies.map((c) => ({
-      name: c.name, value: c.value, domain: c.domain, path: c.path,
-      secure: c.secure, httpOnly: c.httpOnly, sameSite: c.sameSite, expires: c.expires,
-    }));
-
-    const vaultKey = `auth:${getRegistrableDomain(targetDomain)}`;
-    await storeCredential(vaultKey, JSON.stringify({ cookies: storableCookies }));
-    log("auth", `stored ${storableCookies.length} cookies under ${vaultKey}`);
-
-    return { success: true, domain: targetDomain, cookies_stored: storableCookies.length };
-  } finally {
-    // Cleanup handled by Kuri's tab management
   }
+
+  return {
+    success: false,
+    domain: targetDomain,
+    cookies_stored: 0,
+    error: `Login timed out after ${LOGIN_TIMEOUT_MS / 1000}s — no cookies detected in browser`,
+  };
 }
 
 /**

package/runtime-src/execution/index.ts

@@ -596,20 +596,19 @@ export function buildPageArtifactCapture(
   const extracted = extractFromDOM(html, intent);
   if (!extracted.data || extracted.confidence <= 0.2) return {};
   const quality = validateExtractionQuality(extracted.data, extracted.confidence, intent);
-  if (!quality.valid) {
-    return { quality_note: quality.quality_note ?? "low_quality_dom_extraction" };
-  }
   const semanticAssessment = assessIntentResult(extracted.data, intent);
   if (semanticAssessment.verdict === "fail") {
     return { quality_note: semanticAssessment.reason };
   }
+  // Quality gate: low confidence still returns data to the caller (better than
+  // no_endpoints), but marks it so the caller can decide whether to publish.
   const response_schema = inferSchema([extracted.data]);
   const endpoint: EndpointDescriptor = {
     endpoint_id: nanoid(),
     method: "GET",
     url_template: templatizeQueryParams(url),
     idempotency: "safe" as const,
-    verification_status: "verified" as const,
+    verification_status: quality.valid ? "verified" as const : "unverified" as const,
     reliability_score: extracted.confidence,
     description: `Captured page artifact for ${intent}`,
     response_schema,
@@ -637,8 +636,10 @@ export function buildPageArtifactCapture(
         method: extracted.extraction_method,
         confidence: extracted.confidence,
         source: "dom-fallback",
+        ...(quality.quality_note ? { quality_note: quality.quality_note } : {}),
       },
     },
+    ...(!quality.valid ? { quality_note: quality.quality_note } : {}),
   };
 }
 
@@ -1163,9 +1164,27 @@ async function executeBrowserCapture(
     cleanEndpoints.push(canonicalDocumentEndpoint);
   }
 
-  const pageArtifact = captured.html
+  let pageArtifact = captured.html
     ? buildPageArtifactCapture(url, intent, captured.html, authBackedCapture)
     : {};
+
+  // SSR fallback: if Kuri's headless Chrome was bot-detected and served stripped
+  // HTML, the DOM extraction above will fail or return low quality. Try a plain
+  // HTTP fetch — many sites serve full SSR HTML to normal requests.
+  if (!pageArtifact.endpoint) {
+    const kuriHtmlLen = captured.html?.length ?? 0;
+    const ssrFallback = await tryHttpFetch(url, {}, []).catch(() => null);
+    if (ssrFallback && ssrFallback.html.length > kuriHtmlLen * 1.2) {
+      console.log(`[ssr-fallback] Kuri HTML=${kuriHtmlLen}, fetch HTML=${ssrFallback.html.length} — retrying DOM extraction`);
+      const ssrArtifact = buildPageArtifactCapture(ssrFallback.final_url || url, intent, ssrFallback.html, authBackedCapture);
+      if (ssrArtifact.endpoint) {
+        console.log(`[ssr-fallback] success — extracted structured data via plain HTTP fetch`);
+        pageArtifact = ssrArtifact;
+      } else {
+        console.log(`[ssr-fallback] fetch got larger HTML but extraction still failed${ssrArtifact.quality_note ? `: ${ssrArtifact.quality_note}` : ""}`);
+      }
+    }
+  }
   const domArtifactEndpoint = pageArtifact.endpoint;
   const domArtifactResult = pageArtifact.result;
   const inferredOnlyCapture = cleanEndpoints.length > 0 && cleanEndpoints.every((endpoint) => isBundleInferredEndpoint(endpoint));
@@ -1249,7 +1268,8 @@ async function executeBrowserCapture(
         };
       }
 
-    if (pageArtifact.quality_note) {
+    if (pageArtifact.quality_note && !pageArtifact.endpoint) {
+      // Quality gate rejected AND no endpoint — nothing useful extracted
       const trace: ExecutionTrace = stampTrace({
         trace_id: traceId,
         skill_id: skill.skill_id,
@@ -1772,13 +1792,15 @@ export async function executeEndpoint(
 
       // CSRF token auto-detection (bird pattern): many sites require CSRF tokens
       // as both a cookie AND a header. Detect common patterns and replay them.
-      if (!headers["x-csrf-token"] && !headers["x-xsrf-token"]) {
+      if (!headers["x-csrf-token"] && !headers["x-xsrf-token"] && !headers["csrf-token"]) {
         const csrfCookie = cookies.find((c) =>
-          /^(ct0|csrf_token|_csrf|csrftoken|XSRF-TOKEN|_xsrf)$/i.test(c.name)
+          /^(ct0|csrf_token|_csrf|csrftoken|XSRF-TOKEN|_xsrf|JSESSIONID)$/i.test(c.name)
         );
         if (csrfCookie) {
           const v = csrfCookie.value.startsWith('"') && csrfCookie.value.endsWith('"') ? csrfCookie.value.slice(1, -1) : csrfCookie.value;
-          headers["x-csrf-token"] = v;
+          // LinkedIn uses "csrf-token" header derived from JSESSIONID
+          const headerName = csrfCookie.name === "JSESSIONID" ? "csrf-token" : "x-csrf-token";
+          headers[headerName] = v;
         }
       }
     }

package/runtime-src/reverse-engineer/index.ts

@@ -381,8 +381,30 @@ function inferCsrfPlan(req: RawRequest, parsedBody?: unknown): CsrfPlan | undefi
     Object.entries(req.request_headers).map(([key, value]) => [key.toLowerCase(), value]),
   );
   const cookies = parseCookieHeader(headers["cookie"]);
-  const csrfCookieNames = Object.keys(cookies).filter((name) => /^(ct0|csrf_token|_csrf|csrftoken|xsrf-token|_xsrf)$/i.test(name));
-  const headerName = ["x-csrf-token", "x-xsrf-token", "x-csrftoken"].find((name) => typeof headers[name] === "string" && headers[name].length > 0);
+  const csrfCookieNames = Object.keys(cookies).filter((name) => /^(ct0|csrf_token|_csrf|csrftoken|xsrf-token|_xsrf|JSESSIONID)$/i.test(name));
+  const headerName = ["x-csrf-token", "x-xsrf-token", "x-csrftoken", "csrf-token"].find((name) => typeof headers[name] === "string" && headers[name].length > 0);
+
+  // Also detect CSRF by value matching: if any cookie value appears as a header value,
+  // that's a CSRF token pattern regardless of naming convention
+  if (!headerName && csrfCookieNames.length === 0) {
+    for (const [cookieName, cookieValue] of Object.entries(cookies)) {
+      if (!cookieValue || cookieValue.length < 8) continue;
+      const unquoted = cookieValue.startsWith('"') && cookieValue.endsWith('"') ? cookieValue.slice(1, -1) : cookieValue;
+      for (const [hName, hValue] of Object.entries(headers)) {
+        if (hName === "cookie" || hName === "host" || hName === "content-length") continue;
+        const hUnquoted = hValue.startsWith('"') && hValue.endsWith('"') ? hValue.slice(1, -1) : hValue;
+        if (unquoted === hUnquoted && unquoted.length >= 8) {
+          return {
+            source: "cookie",
+            param_name: hName,
+            refresh_on_401: true,
+            extractor_sequence: [cookieName],
+          };
+        }
+      }
+    }
+  }
+
   if (headerName && csrfCookieNames.length > 0) {
     return {
       source: "cookie",