unbrowse 2.0.16 → 2.0.21

package/dist/cli.js

@@ -560,6 +560,7 @@ async function maybeAutoUpdate(metaUrl, overrides = {}) {
 import { closeSync, openSync, readFileSync as readFileSync4, statSync, unlinkSync, writeFileSync as writeFileSync2 } from "node:fs";
 import path3 from "node:path";
 import { spawn } from "node:child_process";
+import { execFileSync } from "node:child_process";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -666,6 +667,34 @@ function clearStalePidFile(pidFile) {
     unlinkSync(pidFile);
   } catch {}
 }
+function findListeningPid(baseUrl) {
+  try {
+    const url = new URL(baseUrl);
+    const port = url.port || (url.protocol === "https:" ? "443" : "80");
+    const output = execFileSync("lsof", ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-t"], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"]
+    }).trim();
+    const pid = Number(output.split(/\s+/).find(Boolean));
+    return Number.isInteger(pid) && pid > 0 ? pid : null;
+  } catch {
+    return null;
+  }
+}
+function readProcessCommand(pid) {
+  try {
+    return execFileSync("ps", ["-o", "command=", "-p", String(pid)], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"]
+    }).trim();
+  } catch {
+    return "";
+  }
+}
+function isLikelyUnbrowseServerProcess(pid) {
+  const command = readProcessCommand(pid);
+  return /\bunbrowse\b|runtime-src\/index\.ts|src\/index\.ts|dist\/index\.js/i.test(command);
+}
 async function stopManagedServer(pid, pidFile, baseUrl) {
   try {
     process.kill(pid, "SIGTERM");
@@ -714,7 +743,13 @@ async function ensureLocalServer(baseUrl, noAutoStart, metaUrl) {
     } else {
       if (existing)
         clearStalePidFile(pidFile);
-      return;
+      const discoveredPid = findListeningPid(baseUrl);
+      if (discoveredPid && isLikelyUnbrowseServerProcess(discoveredPid)) {
+        await stopManagedServer(discoveredPid, pidFile, baseUrl);
+        existing = null;
+      } else {
+        return;
+      }
     }
   }
   if (existing?.pid && isPidAlive(existing.pid)) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "2.0.16",
+  "version": "2.0.21",
   "description": "Reverse-engineer any website into reusable API skills. npm CLI + local engine.",
   "type": "module",
   "bin": {

package/runtime-src/auth/browser-cookies.ts

@@ -27,9 +27,20 @@ export interface BrowserCookie {
   expires: number;
 }
 
+export interface BrowserAuthSourceMeta {
+  family: "chromium" | "firefox";
+  browserName: string;
+  source: string;
+  userDataDir?: string;
+  profile?: string;
+  cookieDbPath?: string;
+  safeStorageService?: string;
+}
+
 export interface ExtractionResult {
   cookies: BrowserCookie[];
   source: string | null;
+  sourceMeta?: BrowserAuthSourceMeta | null;
   warnings: string[];
 }
 
@@ -50,6 +61,13 @@ export interface ExtractBrowserCookiesOptions {
   chromium?: ChromiumCookieSourceOptions;
 }
 
+type ChromiumBrowserCandidate = {
+  name: string;
+  userDataDir: string;
+  safeStorageService: string;
+  bundleId?: string;
+};
+
 // ---------------------------------------------------------------------------
 // Path helpers
 // ---------------------------------------------------------------------------
@@ -66,6 +84,46 @@ function getChromeUserDataDir(): string {
   return join(home, ".config", "google-chrome");
 }
 
+export function extractDefaultBrowserBundleIdFromLaunchServicesData(data: unknown): string | null {
+  const handlers = data && typeof data === "object" && Array.isArray((data as { LSHandlers?: unknown[] }).LSHandlers)
+    ? (data as { LSHandlers: Array<Record<string, unknown>> }).LSHandlers
+    : [];
+  for (const scheme of ["https", "http"]) {
+    const match = handlers.find((entry) => entry.LSHandlerURLScheme === scheme && typeof entry.LSHandlerRoleAll === "string");
+    if (typeof match?.LSHandlerRoleAll === "string" && match.LSHandlerRoleAll.length > 0) {
+      return match.LSHandlerRoleAll;
+    }
+  }
+  return null;
+}
+
+function getMacDefaultBrowserBundleId(): string | null {
+  if (platform() !== "darwin") return null;
+  const plist = join(homedir(), "Library", "Preferences", "com.apple.LaunchServices.com.apple.launchservices.secure.plist");
+  const fallbackPlist = join(homedir(), "Library", "Preferences", "com.apple.LaunchServices", "com.apple.launchservices.secure.plist");
+  const target = existsSync(plist) ? plist : fallbackPlist;
+  if (!existsSync(target)) return null;
+  try {
+    const json = execFileSync("plutil", ["-convert", "json", "-o", "-", target], {
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+    });
+    return extractDefaultBrowserBundleIdFromLaunchServicesData(JSON.parse(json));
+  } catch {
+    return null;
+  }
+}
+
+export function prioritizeChromiumCandidates(
+  sources: ChromiumBrowserCandidate[],
+  preferredBundleId?: string | null,
+): ChromiumBrowserCandidate[] {
+  if (!preferredBundleId) return [...sources];
+  const preferred = sources.find((source) => source.bundleId === preferredBundleId);
+  if (!preferred) return [...sources];
+  return [preferred, ...sources.filter((source) => source !== preferred)];
+}
+
 export function resolveChromiumCookiesPath(opts?: ChromiumCookieSourceOptions): string | null {
   if (opts?.cookieDbPath) {
     return opts.cookieDbPath.replace(/^~\//, homedir() + "/");
@@ -83,6 +141,23 @@ export function resolveChromiumCookiesPath(opts?: ChromiumCookieSourceOptions):
   return candidates.find((candidate) => existsSync(candidate)) ?? candidates[0] ?? null;
 }
 
+function inferChromiumProfileFromPath(
+  dbPath: string,
+  userDataDir?: string,
+): string | undefined {
+  if (!userDataDir) return undefined;
+  const normalizedRoot = userDataDir.replace(/^~\//, homedir() + "/").replace(/\/+$/, "");
+  const normalizedDbPath = dbPath.replace(/\/+$/, "");
+  if (!normalizedDbPath.startsWith(`${normalizedRoot}/`)) return undefined;
+  const rel = normalizedDbPath.slice(normalizedRoot.length + 1);
+  const parts = rel.split("/");
+  if (parts.length < 2) return undefined;
+  if (parts[1] === "Cookies" || (parts[1] === "Network" && parts[2] === "Cookies")) {
+    return parts[0];
+  }
+  return undefined;
+}
+
 function getFirefoxProfilesRoot(): string | null {
   const home = homedir();
   if (platform() === "darwin") {
@@ -262,7 +337,9 @@ export function extractFromChrome(
 ): ExtractionResult {
   return extractFromChromium(domain, {
     profile: opts?.profile,
+    userDataDir: getChromeUserDataDir(),
     browserName: "Chrome",
+    safeStorageService: "Chrome Safe Storage",
   });
 }
 
@@ -276,10 +353,11 @@ export function extractFromChromium(
 
   if (!dbPath || !existsSync(dbPath)) {
     warnings.push(`${sourceLabel} cookies DB not found${dbPath ? ` at ${dbPath}` : ""}`);
-    return { cookies: [], source: null, warnings };
+    return { cookies: [], source: null, sourceMeta: null, warnings };
   }
 
   try {
+    const resolvedProfile = opts?.profile || inferChromiumProfileFromPath(dbPath, opts?.userDataDir);
     const cookies = withTempCopy(dbPath, (tempDb) => {
       const where = buildDomainWhereClause(domain, "host_key");
       const sql = `SELECT name, value, hex(encrypted_value) as ev, host_key, path, is_secure, is_httponly, samesite, expires_utc FROM cookies WHERE ${where};`;
@@ -322,10 +400,21 @@ export function extractFromChromium(
       warnings.push(`No cookies for ${domain} found in ${source}`);
     }
     log("auth", `extracted ${cookies.length} cookies for ${domain} from ${source}`);
-    return { cookies, source: cookies.length > 0 ? source : null, warnings };
+    const sourceMeta: BrowserAuthSourceMeta | null = cookies.length > 0
+      ? {
+          family: "chromium",
+          browserName: sourceLabel,
+          source,
+          ...(opts?.userDataDir ? { userDataDir: opts.userDataDir } : {}),
+          ...(resolvedProfile ? { profile: resolvedProfile } : {}),
+          ...(opts?.cookieDbPath ? { cookieDbPath: dbPath } : {}),
+          ...(opts?.safeStorageService ? { safeStorageService: opts.safeStorageService } : {}),
+        }
+      : null;
+    return { cookies, source: cookies.length > 0 ? source : null, sourceMeta, warnings };
   } catch (err) {
     warnings.push(`${sourceLabel} extraction failed: ${err instanceof Error ? err.message : err}`);
-    return { cookies: [], source: null, warnings };
+    return { cookies: [], source: null, sourceMeta: null, warnings };
   }
 }
 
@@ -343,7 +432,7 @@ export function extractFromFirefox(
 
   if (!dbPath) {
     warnings.push(`${browserLabel} cookies DB not found`);
-    return { cookies: [], source: null, warnings };
+    return { cookies: [], source: null, sourceMeta: null, warnings };
   }
 
   try {
@@ -379,10 +468,18 @@ export function extractFromFirefox(
       warnings.push(`No cookies for ${domain} found in ${source}`);
     }
     log("auth", `extracted ${cookies.length} cookies for ${domain} from ${source}`);
-    return { cookies, source: cookies.length > 0 ? source : null, warnings };
+    const sourceMeta: BrowserAuthSourceMeta | null = cookies.length > 0
+      ? {
+          family: "firefox",
+          browserName: browserLabel,
+          source,
+          ...(opts?.profile ? { profile: opts.profile } : {}),
+        }
+      : null;
+    return { cookies, source: cookies.length > 0 ? source : null, sourceMeta, warnings };
   } catch (err) {
     warnings.push(`${browserLabel} extraction failed: ${err instanceof Error ? err.message : err}`);
-    return { cookies: [], source: null, warnings };
+    return { cookies: [], source: null, sourceMeta: null, warnings };
   }
 }
 
@@ -406,38 +503,27 @@ export function extractBrowserCookies(
     return extractFromChromium(domain, opts.chromium);
   }
 
-  // Try Firefox first (no decryption needed, more reliable)
-  const ff = extractFromFirefox(domain, { profile: opts?.firefoxProfile });
-  if (ff.cookies.length > 0) return ff;
-
-  // If caller provided an explicit Chromium-family source, try that next.
+  // If caller provided an explicit Chromium-family source, try that first.
   if (opts?.chromium?.cookieDbPath || opts?.chromium?.userDataDir) {
     const chromium = extractFromChromium(domain, opts.chromium);
-    chromium.warnings.push(...ff.warnings);
     return chromium;
   }
 
-  // Try Chrome first
-  const chrome = extractFromChrome(domain, { profile: opts?.chromeProfile });
-  if (chrome.cookies.length > 0) {
-    chrome.warnings.push(...ff.warnings);
-    return chrome;
-  }
-
-  // Auto-discover other Chromium-family browsers
   const home = homedir();
-  const chromiumBrowsers: Array<{ name: string; userDataDir: string; safeStorageService: string }> =
+  const chromiumBrowsers: ChromiumBrowserCandidate[] =
     platform() === "darwin"
       ? [
-          { name: "Arc", userDataDir: join(home, "Library", "Application Support", "Arc", "User Data"), safeStorageService: "Arc Safe Storage" },
-          { name: "Dia", userDataDir: join(home, "Library", "Application Support", "Dia", "User Data"), safeStorageService: "Dia Safe Storage" },
-          { name: "Brave", userDataDir: join(home, "Library", "Application Support", "BraveSoftware", "Brave-Browser"), safeStorageService: "Brave Safe Storage" },
-          { name: "Edge", userDataDir: join(home, "Library", "Application Support", "Microsoft Edge"), safeStorageService: "Microsoft Edge Safe Storage" },
-          { name: "Vivaldi", userDataDir: join(home, "Library", "Application Support", "Vivaldi"), safeStorageService: "Vivaldi Safe Storage" },
-          { name: "Chromium", userDataDir: join(home, "Library", "Application Support", "Chromium"), safeStorageService: "Chromium Safe Storage" },
+          { name: "Chrome", userDataDir: getChromeUserDataDir(), safeStorageService: "Chrome Safe Storage", bundleId: "com.google.chrome" },
+          { name: "Arc", userDataDir: join(home, "Library", "Application Support", "Arc", "User Data"), safeStorageService: "Arc Safe Storage", bundleId: "company.thebrowser.Browser" },
+          { name: "Dia", userDataDir: join(home, "Library", "Application Support", "Dia", "User Data"), safeStorageService: "Dia Safe Storage", bundleId: "company.thebrowser.dia" },
+          { name: "Brave", userDataDir: join(home, "Library", "Application Support", "BraveSoftware", "Brave-Browser"), safeStorageService: "Brave Safe Storage", bundleId: "com.brave.Browser" },
+          { name: "Edge", userDataDir: join(home, "Library", "Application Support", "Microsoft Edge"), safeStorageService: "Microsoft Edge Safe Storage", bundleId: "com.microsoft.edgemac" },
+          { name: "Vivaldi", userDataDir: join(home, "Library", "Application Support", "Vivaldi"), safeStorageService: "Vivaldi Safe Storage", bundleId: "com.vivaldi.Vivaldi" },
+          { name: "Chromium", userDataDir: join(home, "Library", "Application Support", "Chromium"), safeStorageService: "Chromium Safe Storage", bundleId: "org.chromium.Chromium" },
         ]
       : platform() === "linux"
         ? [
+            { name: "Chrome", userDataDir: getChromeUserDataDir(), safeStorageService: "Chrome Safe Storage" },
             { name: "Brave", userDataDir: join(home, ".config", "BraveSoftware", "Brave-Browser"), safeStorageService: "Brave Safe Storage" },
             { name: "Edge", userDataDir: join(home, ".config", "microsoft-edge"), safeStorageService: "Microsoft Edge Safe Storage" },
             { name: "Vivaldi", userDataDir: join(home, ".config", "vivaldi"), safeStorageService: "Vivaldi Safe Storage" },
@@ -445,8 +531,33 @@ export function extractBrowserCookies(
           ]
         : [];
 
-  const allWarnings = [...ff.warnings, ...chrome.warnings];
-  for (const browser of chromiumBrowsers) {
+  const preferredBundleId = getMacDefaultBrowserBundleId();
+  const orderedChromiumBrowsers = prioritizeChromiumCandidates(chromiumBrowsers, preferredBundleId);
+
+  const preferredChromium = preferredBundleId ? orderedChromiumBrowsers[0] : null;
+  const accumulatedWarnings: string[] = [];
+  if (preferredChromium?.bundleId === preferredBundleId && existsSync(preferredChromium.userDataDir)) {
+    const preferredResult = extractFromChromium(domain, {
+      userDataDir: preferredChromium.userDataDir,
+      browserName: preferredChromium.name,
+      safeStorageService: preferredChromium.safeStorageService,
+    });
+    if (preferredResult.cookies.length > 0) {
+      return preferredResult;
+    }
+    accumulatedWarnings.push(...preferredResult.warnings);
+  }
+
+  // Try Firefox next (no decryption needed, more reliable when it actually has the session)
+  const ff = extractFromFirefox(domain, { profile: opts?.firefoxProfile });
+  if (ff.cookies.length > 0) {
+    ff.warnings.push(...accumulatedWarnings);
+    return ff;
+  }
+
+  const allWarnings = [...accumulatedWarnings, ...ff.warnings];
+  for (const browser of orderedChromiumBrowsers) {
+    if (browser.bundleId && browser.bundleId === preferredBundleId) continue;
     if (!existsSync(browser.userDataDir)) continue;
     const result = extractFromChromium(domain, {
       userDataDir: browser.userDataDir,
@@ -474,5 +585,5 @@ export function extractBrowserCookies(
     allWarnings.push(...zenResult.warnings);
   }
 
-  return { cookies: [], source: null, warnings: allWarnings };
+  return { cookies: [], source: null, sourceMeta: null, warnings: allWarnings };
 }

package/runtime-src/auth/index.ts

@@ -6,6 +6,7 @@ import { log } from "../logger.js";
 import path from "node:path";
 import os from "node:os";
 import fs from "node:fs";
+import type { BrowserAuthSourceMeta } from "./browser-cookies.js";
 
 const LOGIN_TIMEOUT_MS = 300_000;
 const POLL_INTERVAL_MS = 2_000;
@@ -30,6 +31,18 @@ export interface StoredAuthBundle {
   cookies: AuthCookie[];
   headers: Record<string, string>;
   source_keys: string[];
+  source_meta?: BrowserAuthSourceMeta | null;
+}
+
+export function storedAuthNeedsBrowserRefresh(bundle: StoredAuthBundle | null | undefined): boolean {
+  if (!bundle) return true;
+  if (bundle.cookies.length === 0 && Object.keys(bundle.headers).length === 0) return true;
+  const sourceMeta = bundle.source_meta;
+  if (!sourceMeta) return true;
+  if (sourceMeta.family === "chromium" && !sourceMeta.userDataDir && !sourceMeta.cookieDbPath) {
+    return true;
+  }
+  return false;
 }
 
 /**
@@ -118,7 +131,7 @@ export async function extractBrowserAuth(
   const vaultKey = `auth:${getRegistrableDomain(domain)}`;
   await storeCredential(
     vaultKey,
-    JSON.stringify({ cookies: storableCookies })
+    JSON.stringify({ cookies: storableCookies, source_meta: result.sourceMeta ?? null })
   );
 
   log("auth", `stored ${storableCookies.length} cookies for ${domain} (key: ${vaultKey}) from ${result.source}`);
@@ -172,6 +185,7 @@ export async function getStoredAuthBundle(
   const cookies: AuthCookie[] = [];
   const headers: Record<string, string> = {};
   const source_keys: string[] = [];
+  let source_meta: BrowserAuthSourceMeta | null = null;
 
   for (const key of getStoredAuthKeys(domain)) {
     const stored = await getCredential(key);
@@ -180,6 +194,7 @@ export async function getStoredAuthBundle(
       const parsed = JSON.parse(stored) as {
         cookies?: AuthCookie[];
         headers?: Record<string, string>;
+        source_meta?: BrowserAuthSourceMeta | null;
       };
       const rawCookies = parsed.cookies ?? [];
       const validCookies = filterExpired(rawCookies);
@@ -198,6 +213,7 @@ export async function getStoredAuthBundle(
       }
       if ((validCookies.length > 0 || Object.keys(parsedHeaders).length > 0) && !source_keys.includes(key)) {
         source_keys.push(key);
+        if (!source_meta && parsed.source_meta) source_meta = parsed.source_meta;
       }
     } catch {
       continue;
@@ -205,7 +221,7 @@ export async function getStoredAuthBundle(
   }
 
   if (cookies.length === 0 && Object.keys(headers).length === 0) return null;
-  return { cookies, headers, source_keys };
+  return { cookies, headers, source_keys, source_meta };
 }
 
 export async function findStoredAuthReference(domain: string): Promise<string | null> {
@@ -234,12 +250,14 @@ export async function getAuthCookies(
   domain: string,
   opts?: { autoExtract?: boolean }
 ): Promise<AuthCookie[] | null> {
-  const vaultCookies = await getStoredAuth(domain);
-  if (vaultCookies && vaultCookies.length > 0) return vaultCookies;
+  const bundle = await getStoredAuthBundle(domain);
+  if (bundle && bundle.cookies.length > 0 && !storedAuthNeedsBrowserRefresh(bundle)) {
+    return bundle.cookies;
+  }
 
-  if (opts?.autoExtract === false) return null;
+  if (opts?.autoExtract === false) return bundle?.cookies ?? null;
 
-  log("auth", `no vault cookies for ${domain} — auto-extracting from browser`);
+  log("auth", `${bundle ? "stored auth lacks usable browser source metadata" : "no vault cookies"} for ${domain} — auto-extracting from browser`);
   try {
     const result = await extractBrowserAuth(domain);
     if (result.success && result.cookies_stored > 0) {
@@ -249,7 +267,7 @@ export async function getAuthCookies(
     log("auth", `browser auto-extract failed for ${domain}: ${err instanceof Error ? err.message : err}`);
   }
 
-  return null;
+  return bundle?.cookies ?? null;
 }
 
 /**