unbrowse 2.0.21 → 2.0.22

package/dist/cli.js

@@ -325,6 +325,7 @@ import { existsSync as existsSync2, mkdirSync as mkdirSync2, realpathSync } from
 import os from "node:os";
 import path from "node:path";
 import { createRequire } from "node:module";
+import { execFileSync } from "node:child_process";
 import { fileURLToPath } from "node:url";
 function getModuleDir(metaUrl) {
   return path.dirname(fileURLToPath(metaUrl));
@@ -349,19 +350,32 @@ function resolveSiblingEntrypoint(metaUrl, basename) {
   const file = fileURLToPath(metaUrl);
   return path.join(path.dirname(file), `${basename}${path.extname(file) || ".js"}`);
 }
-function runtimeArgsForEntrypoint(metaUrl, entrypoint) {
+function resolveBinaryOnPath(name) {
+  const checker = process.platform === "win32" ? "where" : "which";
+  try {
+    const output = execFileSync(checker, [name], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+    const match = output.split(/\r?\n/).map((line) => line.trim()).find(Boolean);
+    return match || null;
+  } catch {
+    return null;
+  }
+}
+function runtimeInvocationForEntrypoint(metaUrl, entrypoint) {
   if (path.extname(entrypoint) !== ".ts")
-    return [entrypoint];
+    return { command: process.execPath, args: [entrypoint] };
   if (process.versions.bun)
-    return [entrypoint];
+    return { command: process.execPath, args: [entrypoint] };
+  const bunBinary = process.env.BUN_BIN || resolveBinaryOnPath("bun");
+  if (bunBinary)
+    return { command: bunBinary, args: [entrypoint] };
   try {
     const req = createRequire(metaUrl);
     const tsxPkg = req.resolve("tsx/package.json");
     const tsxLoader = path.join(path.dirname(tsxPkg), "dist", "loader.mjs");
     if (existsSync2(tsxLoader))
-      return ["--import", tsxLoader, entrypoint];
+      return { command: process.execPath, args: ["--import", tsxLoader, entrypoint] };
   } catch {}
-  return ["--import", "tsx", entrypoint];
+  return { command: process.execPath, args: ["--import", "tsx", entrypoint] };
 }
 function getUnbrowseHome() {
   return path.join(os.homedir(), ".unbrowse");
@@ -560,7 +574,7 @@ async function maybeAutoUpdate(metaUrl, overrides = {}) {
 import { closeSync, openSync, readFileSync as readFileSync4, statSync, unlinkSync, writeFileSync as writeFileSync2 } from "node:fs";
 import path3 from "node:path";
 import { spawn } from "node:child_process";
-import { execFileSync } from "node:child_process";
+import { execFileSync as execFileSync2 } from "node:child_process";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -671,7 +685,7 @@ function findListeningPid(baseUrl) {
   try {
     const url = new URL(baseUrl);
     const port = url.port || (url.protocol === "https:" ? "443" : "80");
-    const output = execFileSync("lsof", ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-t"], {
+    const output = execFileSync2("lsof", ["-nP", `-iTCP:${port}`, "-sTCP:LISTEN", "-t"], {
       encoding: "utf8",
       stdio: ["ignore", "pipe", "ignore"]
     }).trim();
@@ -683,7 +697,7 @@ function findListeningPid(baseUrl) {
 }
 function readProcessCommand(pid) {
   try {
-    return execFileSync("ps", ["-o", "command=", "-p", String(pid)], {
+    return execFileSync2("ps", ["-o", "command=", "-p", String(pid)], {
       encoding: "utf8",
       stdio: ["ignore", "pipe", "ignore"]
     }).trim();
@@ -791,7 +805,8 @@ async function ensureLocalServer(baseUrl, noAutoStart, metaUrl) {
     const logFile = getServerAutostartLogFile();
     ensureDir(path3.dirname(logFile));
     const logFd = openSync(logFile, "a");
-    const child = spawn(process.execPath, runtimeArgsForEntrypoint(metaUrl, entrypoint), {
+    const runtime = runtimeInvocationForEntrypoint(metaUrl, entrypoint);
+    const child = spawn(runtime.command, runtime.args, {
       cwd: packageRoot,
       detached: true,
       stdio: ["ignore", logFd, logFd],

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "2.0.21",
+  "version": "2.0.22",
   "description": "Reverse-engineer any website into reusable API skills. npm CLI + local engine.",
   "type": "module",
   "bin": {

package/runtime-src/auth/profile-context.ts

@@ -38,6 +38,25 @@ export interface LaunchedProfileContext {
   tempDir: string;
 }
 
+async function waitForChildExit(child: ChildProcess, timeoutMs = 2_000): Promise<void> {
+  if (child.exitCode !== null || child.killed) return;
+  await new Promise<void>((resolve) => {
+    const timer = setTimeout(resolve, timeoutMs);
+    child.once("exit", () => {
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+}
+
+function removeTempDirQuietly(dir: string): void {
+  try {
+    rmSync(dir, { recursive: true, force: true });
+  } catch {
+    // best-effort cleanup; do not fail captures on temp profile removal
+  }
+}
+
 function resolveChromiumBinary(browserName: string): string | null {
   const macos = new Map<string, string>([
     ["Chrome", "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
@@ -296,13 +315,14 @@ export async function launchChromiumProfileContext(meta: BrowserAuthSourceMeta):
     };
   } catch (error) {
     try { child.kill("SIGTERM"); } catch {}
-    rmSync(tempDir, { recursive: true, force: true });
+    removeTempDirQuietly(tempDir);
     throw error;
   }
 }
 
-export function cleanupProfileContext(ctx: LaunchedProfileContext | null | undefined): void {
+export async function cleanupProfileContext(ctx: LaunchedProfileContext | null | undefined): Promise<void> {
   if (!ctx) return;
   try { ctx.child.kill("SIGTERM"); } catch {}
-  rmSync(ctx.tempDir, { recursive: true, force: true });
+  await waitForChildExit(ctx.child);
+  removeTempDirQuietly(ctx.tempDir);
 }

package/runtime-src/capture/index.ts

@@ -268,6 +268,20 @@ export function blockedAppShellErrorCode(
   return hasAuth ? "blocked_app_shell" : "auth_required";
 }
 
+export function shouldShortCircuitEmbeddedPayloadCapture(url: string, intent: string | undefined, html?: string): boolean {
+  if (!html) return false;
+  const lowerIntent = intent?.toLowerCase() ?? "";
+  if (
+    /linkedin\.com/i.test(url) &&
+    /\/feed(?:\/|$)/i.test(url) &&
+    /\b(feed|timeline|stream|post|posts|update|updates|home)\b/.test(lowerIntent) &&
+    /voyagerFeedDashMainFeed/.test(html)
+  ) {
+    return true;
+  }
+  return false;
+}
+
 function shouldRetryEphemeralProfileError(error: unknown): boolean {
   const message = error instanceof Error ? error.message : String(error ?? "");
   return /persistentcontext|target page, context or browser has been closed|browser has been closed|page has been closed/i.test(message);
@@ -930,16 +944,23 @@ export async function captureSession(
       }
       await kuri.stop();
       kuri.useExternalChrome(browserCdpBaseUrl(profileCtx.cdpUrl), { child: profileCtx.child, tempDir: profileCtx.tempDir });
+      let nestedResult: CaptureResult | null = null;
       try {
-        return await captureSession(url, undefined, undefined, intent, {
+        nestedResult = await captureSession(url, undefined, undefined, intent, {
           ...options,
           forceEphemeral: true,
           usedProfileContext: true,
           preferExistingTab,
           authStrategy: "header-replay",
         });
+        return nestedResult;
       } finally {
-        await kuri.stop();
+        try {
+          await kuri.stop();
+        } catch (stopErr) {
+          log("capture", `profile-context cleanup failed for ${url}: ${stopErr instanceof Error ? stopErr.message : String(stopErr)}`);
+          if (!nestedResult) throw stopErr;
+        }
       }
     } catch (attachErr) {
       log("capture", `forced profile context failed for ${url}: ${attachErr instanceof Error ? attachErr.message : String(attachErr)}`);
@@ -1081,6 +1102,40 @@ export async function captureSession(
       await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT);
     } catch { /* page may not be ready */ }
 
+    // For pages that embed the task payload directly in the HTML, return before
+    // the longer network/intercept wait. This avoids losing useful captures to
+    // later browser-engine instability on auth-gated SPAs like LinkedIn feed.
+    try {
+      await sleep(1_500, signal);
+      throwIfAborted(signal);
+      const earlyHtml = await kuri.getPageHtml(tabId);
+      if (shouldShortCircuitEmbeddedPayloadCapture(url, intent, earlyHtml)) {
+        let final_url = url;
+        try {
+          const rawUrl = await kuri.getCurrentUrl(tabId);
+          final_url = typeof rawUrl === "string" ? rawUrl : String(rawUrl ?? url);
+          try { new URL(final_url); } catch { final_url = url; }
+        } catch {
+          final_url = url;
+        }
+        lastHtml = earlyHtml;
+        const rawCookies = await extractCookiesFromPage(tabId, url);
+        const sessionCookies = filterFirstPartySessionCookies(rawCookies, url, final_url);
+        log("capture", `short-circuiting embedded payload capture for ${url}`);
+        return {
+          requests: [],
+          har_lineage_id: nanoid(),
+          domain,
+          cookies: sessionCookies,
+          final_url,
+          html: earlyHtml,
+          js_bundles: new Map(),
+        };
+      }
+    } catch {
+      // fall through to the longer capture path
+    }
+
     // Build response bodies map from intercepted requests
     const responseBodies = new Map<string, string>();
     const jsBundleBodies = new Map<string, string>();

package/runtime-src/kuri/client.ts

@@ -65,6 +65,18 @@ let externalChromeOverride: {
   previousAttach?: string;
 } | null = null;
 
+async function waitForChildExit(child: ChildProcess | null | undefined, timeoutMs = 2_000): Promise<void> {
+  if (!child) return;
+  if (child.exitCode !== null || child.killed) return;
+  await new Promise<void>((resolve) => {
+    const timer = setTimeout(resolve, timeoutMs);
+    child.once("exit", () => {
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+}
+
 function kuriBinaryName(): string {
   return process.platform === "win32" ? "kuri.exe" : "kuri";
 }
@@ -426,7 +438,12 @@ export async function stop(): Promise<void> {
       // ignore
     }
     if (externalChromeOverride.tempDir) {
-      rmSync(externalChromeOverride.tempDir, { recursive: true, force: true });
+      await waitForChildExit(externalChromeOverride.child);
+      try {
+        rmSync(externalChromeOverride.tempDir, { recursive: true, force: true });
+      } catch {
+        // best-effort cleanup; don't fail the caller on temp profile removal
+      }
     }
     if (externalChromeOverride.previousCdpUrl == null) delete process.env.CDP_URL;
     else process.env.CDP_URL = externalChromeOverride.previousCdpUrl;

package/runtime-src/runtime/local-server.ts

@@ -2,7 +2,7 @@ import { closeSync, openSync, readFileSync, statSync, unlinkSync, writeFileSync
 import path from "node:path";
 import { spawn } from "node:child_process";
 import { execFileSync } from "node:child_process";
-import { ensureDir, getPackageRoot, getServerAutostartLogFile, getServerPidFile, resolveSiblingEntrypoint, runtimeArgsForEntrypoint } from "./paths.js";
+import { ensureDir, getPackageRoot, getServerAutostartLogFile, getServerPidFile, resolveSiblingEntrypoint, runtimeInvocationForEntrypoint } from "./paths.js";
 import { CODE_HASH } from "../version.js";
 
 type PidState = {
@@ -222,7 +222,8 @@ export async function ensureLocalServer(baseUrl: string, noAutoStart: boolean, m
     const logFile = getServerAutostartLogFile();
     ensureDir(path.dirname(logFile));
     const logFd = openSync(logFile, "a");
-    const child = spawn(process.execPath, runtimeArgsForEntrypoint(metaUrl, entrypoint), {
+    const runtime = runtimeInvocationForEntrypoint(metaUrl, entrypoint);
+    const child = spawn(runtime.command, runtime.args, {
       cwd: packageRoot,
       detached: true,
       stdio: ["ignore", logFd, logFd],

package/runtime-src/runtime/paths.ts

@@ -2,6 +2,7 @@ import { existsSync, mkdirSync, realpathSync } from "node:fs";
 import os from "node:os";
 import path from "node:path";
 import { createRequire } from "node:module";
+import { execFileSync } from "node:child_process";
 import { fileURLToPath } from "node:url";
 
 export function getModuleDir(metaUrl: string): string {
@@ -29,20 +30,38 @@ export function resolveSiblingEntrypoint(metaUrl: string, basename: string): str
   return path.join(path.dirname(file), `${basename}${path.extname(file) || ".js"}`);
 }
 
-export function runtimeArgsForEntrypoint(metaUrl: string, entrypoint: string): string[] {
-  if (path.extname(entrypoint) !== ".ts") return [entrypoint];
-  if (process.versions.bun) return [entrypoint];
+function resolveBinaryOnPath(name: string): string | null {
+  const checker = process.platform === "win32" ? "where" : "which";
+  try {
+    const output = execFileSync(checker, [name], { encoding: "utf8", stdio: ["ignore", "pipe", "ignore"] });
+    const match = output.split(/\r?\n/).map((line) => line.trim()).find(Boolean);
+    return match || null;
+  } catch {
+    return null;
+  }
+}
+
+export function runtimeInvocationForEntrypoint(metaUrl: string, entrypoint: string): { command: string; args: string[] } {
+  if (path.extname(entrypoint) !== ".ts") return { command: process.execPath, args: [entrypoint] };
+  if (process.versions.bun) return { command: process.execPath, args: [entrypoint] };
+
+  const bunBinary = process.env.BUN_BIN || resolveBinaryOnPath("bun");
+  if (bunBinary) return { command: bunBinary, args: [entrypoint] };
 
   try {
     const req = createRequire(metaUrl);
     const tsxPkg = req.resolve("tsx/package.json");
     const tsxLoader = path.join(path.dirname(tsxPkg), "dist", "loader.mjs");
-    if (existsSync(tsxLoader)) return ["--import", tsxLoader, entrypoint];
+    if (existsSync(tsxLoader)) return { command: process.execPath, args: ["--import", tsxLoader, entrypoint] };
   } catch {
     // fall through to bare specifier
   }
 
-  return ["--import", "tsx", entrypoint];
+  return { command: process.execPath, args: ["--import", "tsx", entrypoint] };
+}
+
+export function runtimeArgsForEntrypoint(metaUrl: string, entrypoint: string): string[] {
+  return runtimeInvocationForEntrypoint(metaUrl, entrypoint).args;
 }
 
 export function isMainModule(metaUrl: string): boolean {