unbrowse 3.1.0 → 3.2.0

package/runtime-src/runtime/local-server.ts

@@ -1,311 +0,0 @@
-import { openSync, readFileSync, unlinkSync, writeFileSync } from "node:fs";
-import path from "node:path";
-import { spawn } from "node:child_process";
-import { ensureDir, getPackageRoot, getServerAutostartLogFile, getServerPidFile, resolveSiblingEntrypoint, runtimeArgsForEntrypoint } from "./paths.js";
-import { LocalSupervisor } from "./supervisor.js";
-import { CODE_HASH, PACKAGE_VERSION } from "../version.js";
-
-type PidState = {
-  pid: number;
-  base_url: string;
-  started_at: string;
-  entrypoint: string;
-  version?: string;
-  code_hash?: string;
-  restart_count?: number;
-};
-
-type ServerHealth = {
-  status?: string;
-  package_version?: string;
-  code_hash?: string;
-};
-
-export function isServerVersionMismatch(
-  runningVersion: string | undefined,
-  installedVersion: string,
-  runningCodeHash?: string,
-  installedCodeHash?: string,
-): boolean {
-  return (!!runningVersion && runningVersion !== installedVersion)
-    || (!!runningCodeHash && !!installedCodeHash && runningCodeHash !== installedCodeHash);
-}
-
-async function fetchServerHealth(baseUrl: string, timeoutMs = 2_000): Promise<ServerHealth | null> {
-  try {
-    const res = await fetch(`${baseUrl}/health`, { signal: AbortSignal.timeout(timeoutMs) });
-    if (!res.ok) return null;
-    return await res.json() as ServerHealth;
-  } catch {
-    return null;
-  }
-}
-
-async function isServerHealthy(baseUrl: string, timeoutMs = 2_000): Promise<boolean> {
-  return !!(await fetchServerHealth(baseUrl, timeoutMs));
-}
-
-async function waitForHealthy(baseUrl: string, timeoutMs: number): Promise<boolean> {
-  const start = Date.now();
-  while (Date.now() - start < timeoutMs) {
-    if (await isServerHealthy(baseUrl)) return true;
-    await new Promise((resolve) => setTimeout(resolve, 500));
-  }
-  return false;
-}
-
-function isPidAlive(pid: number): boolean {
-  try {
-    process.kill(pid, 0);
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-function readPidState(pidFile: string): PidState | null {
-  try {
-    return JSON.parse(readFileSync(pidFile, "utf-8")) as PidState;
-  } catch {
-    return null;
-  }
-}
-
-function clearStalePidFile(pidFile: string): void {
-  try {
-    unlinkSync(pidFile);
-  } catch {
-    // ignore
-  }
-}
-
-function deriveListenEnv(baseUrl: string): Record<string, string> {
-  const url = new URL(baseUrl);
-  const host = !url.hostname || url.hostname === "localhost" ? "127.0.0.1" : url.hostname;
-  const port = url.port || (url.protocol === "https:" ? "443" : "80");
-  return { HOST: host, PORT: port, UNBROWSE_URL: baseUrl };
-}
-
-function isCompiledBinary(): boolean {
-  return !!(process.versions.bun && !process.argv[1]?.match(/\.(ts|js|mjs)$/));
-}
-
-export function getServerSpawnSpec(metaUrl: string, entrypoint = resolveSiblingEntrypoint(metaUrl, "index")): {
-  command: string;
-  args: string[];
-  cwd: string;
-  recordedEntrypoint: string;
-} {
-  if (isCompiledBinary()) {
-    return {
-      command: process.execPath,
-      args: ["serve"],
-      cwd: process.cwd(),
-      recordedEntrypoint: `${process.execPath} serve`,
-    };
-  }
-
-  return {
-    command: process.execPath,
-    args: runtimeArgsForEntrypoint(metaUrl, entrypoint),
-    cwd: getPackageRoot(metaUrl),
-    recordedEntrypoint: entrypoint,
-  };
-}
-
-const MAX_RESTART_ATTEMPTS = 3;
-const RESTART_BACKOFF_MS = 2_000;
-
-/**
- * Spawn the local server as a detached child process.
- * Returns the PidState written to the pid file.
- */
-function spawnServer(
-  baseUrl: string,
-  metaUrl: string,
-  pidFile: string,
-  restartCount = 0,
-): PidState {
-  const entrypoint = resolveSiblingEntrypoint(metaUrl, "index");
-  const spawnSpec = getServerSpawnSpec(metaUrl, entrypoint);
-  const logFile = getServerAutostartLogFile();
-  ensureDir(path.dirname(logFile));
-  const logFd = openSync(logFile, "a");
-  const child = spawn(spawnSpec.command, spawnSpec.args, {
-    cwd: spawnSpec.cwd,
-    detached: true,
-    stdio: ["ignore", logFd, logFd],
-    env: {
-      ...process.env,
-      ...deriveListenEnv(baseUrl),
-      UNBROWSE_NON_INTERACTIVE: process.env.UNBROWSE_NON_INTERACTIVE || "1",
-      UNBROWSE_TOS_ACCEPTED: process.env.UNBROWSE_TOS_ACCEPTED || "1",
-      UNBROWSE_PID_FILE: pidFile,
-    },
-  });
-  child.unref();
-
-  const state: PidState = {
-    pid: child.pid!,
-    base_url: baseUrl,
-    started_at: new Date().toISOString(),
-    entrypoint: spawnSpec.recordedEntrypoint,
-    version: PACKAGE_VERSION,
-    code_hash: CODE_HASH,
-    restart_count: restartCount,
-  };
-  writeFileSync(pidFile, JSON.stringify(state, null, 2));
-  return state;
-}
-
-/** Shared supervisor instance used for health checking. */
-const supervisor = new LocalSupervisor();
-
-export { supervisor };
-
-export async function ensureLocalServer(baseUrl: string, noAutoStart: boolean, metaUrl: string): Promise<void> {
-  const installedVersion = PACKAGE_VERSION;
-  const initialHealth = await fetchServerHealth(baseUrl);
-  if (initialHealth) {
-    const runningVersion = initialHealth.package_version;
-    const runningCodeHash = initialHealth.code_hash;
-    if (isServerVersionMismatch(runningVersion, installedVersion, runningCodeHash, CODE_HASH)) {
-      const versionInfo = checkServerVersion(baseUrl, metaUrl, {
-        runningVersion,
-        runningCodeHash,
-      });
-      if (versionInfo?.needs_restart) {
-        stopServer(baseUrl);
-        await new Promise((resolve) => setTimeout(resolve, 1_000));
-      } else {
-        throw new Error(
-          `Server runtime mismatch on ${baseUrl}: running ${runningVersion ?? "unknown"} (${runningCodeHash ?? "unknown"}), installed ${installedVersion} (${CODE_HASH}). Run "unbrowse restart" or stop the stale server bound to that port.`,
-        );
-      }
-    } else {
-    // Server already healthy — ensure supervisor state reflects this
-      if (!supervisor.isRunning()) await supervisor.start();
-      return;
-    }
-  }
-
-  const pidFile = getServerPidFile(baseUrl);
-  const existing = readPidState(pidFile);
-
-  if (existing?.pid && isPidAlive(existing.pid)) {
-    // Process alive but not healthy — wait then try supervisor restart
-    if (await waitForHealthy(baseUrl, 15_000)) {
-      if (!supervisor.isRunning()) await supervisor.start();
-      return;
-    }
-    // Still unhealthy after wait — kill stale process and restart
-    try { process.kill(existing.pid, "SIGTERM"); } catch { /* ignore */ }
-    await new Promise((r) => setTimeout(r, 1_000));
-    clearStalePidFile(pidFile);
-    if (supervisor.isRunning()) await supervisor.stop();
-  } else if (existing) {
-    clearStalePidFile(pidFile);
-  }
-
-  if (noAutoStart) {
-    throw new Error("Server not running and auto-start disabled (--no-auto-start).");
-  }
-
-  // Spawn with supervisor retry
-  for (let attempt = 0; attempt <= MAX_RESTART_ATTEMPTS; attempt++) {
-    spawnServer(baseUrl, metaUrl, pidFile, attempt);
-
-    if (await waitForHealthy(baseUrl, 30_000)) {
-      await supervisor.start();
-      return;
-    }
-
-    // Failed to start — clear and retry with backoff
-    const state = readPidState(pidFile);
-    if (state?.pid) {
-      try { process.kill(state.pid, "SIGTERM"); } catch { /* ignore */ }
-    }
-    clearStalePidFile(pidFile);
-
-    if (attempt < MAX_RESTART_ATTEMPTS) {
-      const backoff = RESTART_BACKOFF_MS * (attempt + 1);
-      await new Promise((r) => setTimeout(r, backoff));
-    }
-  }
-
-  const logFile = getServerAutostartLogFile();
-  throw new Error(`Server failed to start after ${MAX_RESTART_ATTEMPTS + 1} attempts. Check ${logFile}`);
-}
-
-/**
- * Check if the running server version matches the installed CLI version.
- * Returns null if no server is running, or runtime/version details.
- */
-export function checkServerVersion(
-  baseUrl: string,
-  metaUrl: string,
-): {
-  running: string;
-  installed: string;
-  running_code_hash?: string;
-  installed_code_hash: string;
-  needs_restart: boolean;
-} | null;
-export function checkServerVersion(
-  baseUrl: string,
-  metaUrl: string,
-  healthOverride?: {
-    runningVersion?: string;
-    runningCodeHash?: string;
-  },
-): {
-  running: string;
-  installed: string;
-  running_code_hash?: string;
-  installed_code_hash: string;
-  needs_restart: boolean;
-} | null {
-  const pidFile = getServerPidFile(baseUrl);
-  const state = readPidState(pidFile);
-  if (!state) return null;
-  const installed = PACKAGE_VERSION;
-  const running = healthOverride?.runningVersion ?? state.version ?? "unknown";
-  const runningCodeHash = healthOverride?.runningCodeHash ?? state.code_hash;
-  return {
-    running,
-    installed,
-    ...(runningCodeHash ? { running_code_hash: runningCodeHash } : {}),
-    installed_code_hash: CODE_HASH,
-    needs_restart: isServerVersionMismatch(running, installed, runningCodeHash, CODE_HASH),
-  };
-}
-
-/**
- * Stop the running server gracefully.
- * Returns true if a server was stopped, false if none was running.
- */
-export function stopServer(baseUrl: string): boolean {
-  const pidFile = getServerPidFile(baseUrl);
-  const state = readPidState(pidFile);
-  if (!state?.pid) return false;
-  try {
-    process.kill(state.pid, "SIGTERM");
-    clearStalePidFile(pidFile);
-    // Synchronously mark supervisor as stopped (fire-and-forget the async stop)
-    if (supervisor.isRunning()) void supervisor.stop();
-    return true;
-  } catch {
-    clearStalePidFile(pidFile);
-    return false;
-  }
-}
-
-/**
- * Restart the local server (stop + start).
- * Used after CLI upgrades to pick up new code.
- */
-export async function restartServer(baseUrl: string, metaUrl: string): Promise<void> {
-  stopServer(baseUrl);
-  await new Promise((r) => setTimeout(r, 1_000));
-  await ensureLocalServer(baseUrl, false, metaUrl);
-}

package/runtime-src/runtime/paths.ts

@@ -1,99 +0,0 @@
-import { existsSync, mkdirSync, realpathSync } from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { createRequire } from "node:module";
-import { fileURLToPath, pathToFileURL } from "node:url";
-
-export function getModuleDir(metaUrl: string): string {
-  return path.dirname(fileURLToPath(metaUrl));
-}
-
-export function getPackageRoot(metaUrl: string): string {
-  if (process.env.UNBROWSE_PACKAGE_ROOT) return process.env.UNBROWSE_PACKAGE_ROOT;
-  let dir = getModuleDir(metaUrl);
-  const root = path.parse(dir).root;
-  while (dir !== root) {
-    if (existsSync(path.join(dir, "package.json"))) return dir;
-    dir = path.dirname(dir);
-  }
-  return getModuleDir(metaUrl);
-}
-
-export function resolveSiblingEntrypoint(metaUrl: string, basename: string): string {
-  const file = fileURLToPath(metaUrl);
-  return path.join(path.dirname(file), `${basename}${path.extname(file) || ".js"}`);
-}
-
-export function isBundledVirtualEntrypoint(entrypoint: string): boolean {
-  return entrypoint.startsWith("/$bunfs/");
-}
-
-export function runtimeArgsForEntrypoint(metaUrl: string, entrypoint: string): string[] {
-  if (path.extname(entrypoint) !== ".ts") return [entrypoint];
-  if (process.versions.bun) return [entrypoint];
-
-  try {
-    const req = createRequire(metaUrl);
-    const tsxPkg = req.resolve("tsx/package.json");
-    const tsxLoader = path.join(path.dirname(tsxPkg), "dist", "loader.mjs");
-    if (existsSync(tsxLoader)) return ["--import", pathToFileURL(tsxLoader).href, entrypoint];
-  } catch {
-    // fall through to bare specifier
-  }
-
-  return ["--import", "tsx", entrypoint];
-}
-
-export function isMainModule(metaUrl: string): boolean {
-  const entry = process.argv[1];
-  if (!entry) return false;
-
-  const modulePath = fileURLToPath(metaUrl);
-  try {
-    return realpathSync(entry) === realpathSync(modulePath);
-  } catch {
-    return path.resolve(entry) === path.resolve(modulePath);
-  }
-}
-
-export function getUnbrowseHome(): string {
-  return path.join(os.homedir(), ".unbrowse");
-}
-
-export function ensureDir(dir: string): string {
-  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
-  return dir;
-}
-
-export function getLogsDir(): string {
-  return ensureDir(path.join(getUnbrowseHome(), "logs"));
-}
-
-export function getRunDir(): string {
-  return ensureDir(process.env.UNBROWSE_RUN_DIR || path.join(getUnbrowseHome(), "run"));
-}
-
-function sanitizeSegment(value: string): string {
-  return value.replace(/[^a-zA-Z0-9.-]+/g, "_");
-}
-
-export function getServerPidFile(baseUrl: string): string {
-  const url = new URL(baseUrl);
-  const port = url.port || (url.protocol === "https:" ? "443" : "80");
-  const host = sanitizeSegment(url.hostname || "127.0.0.1");
-  return path.join(getRunDir(), `server-${host}-${port}.json`);
-}
-
-export function getServerAutostartLogFile(): string {
-  return path.join(getLogsDir(), "server-autostart.log");
-}
-
-export function isManagedSkillInstall(metaUrl: string): boolean {
-  const managedDir = process.env.UNBROWSE_SKILL_DIR || path.join(os.homedir(), ".agents", "skills", "unbrowse");
-  const pkgRoot = getPackageRoot(metaUrl);
-  try {
-    return realpathSync(pkgRoot) === realpathSync(managedDir);
-  } catch {
-    return false;
-  }
-}

package/runtime-src/runtime/setup.ts

@@ -1,251 +0,0 @@
-import { execFileSync } from "node:child_process";
-import { existsSync, mkdirSync, writeFileSync } from "node:fs";
-import os from "node:os";
-import path from "node:path";
-import { ensureDir } from "./paths.js";
-import { findKuriBinary, getKuriSourceCandidates } from "../kuri/client.js";
-import { detectHostEnvironment, type HostEnvironment } from "./browser-host.js";
-import { log } from "../logger.js";
-import { checkWalletConfigured, type WalletCheckResult } from "../payments/wallet.js";
-import { configureUpdateHintHooks, saveInstallSource, type UpdateHookStatus } from "./update-hints.js";
-
-export type SetupScope = "auto" | "global" | "project" | "off";
-
-export type SetupReport = {
-  os: {
-    platform: NodeJS.Platform;
-    release: string;
-    arch: string;
-  };
-  host_environment: HostEnvironment;
-  package_managers: {
-    npm: boolean;
-    npx: boolean;
-    bun: boolean;
-    brew: boolean;
-  };
-  browser_engine: {
-    installed: boolean;
-    action: "already-installed" | "installed" | "failed" | "skipped";
-    message?: string;
-  };
-  opencode: {
-    detected: boolean;
-    action: "installed" | "updated" | "skipped" | "not-detected";
-    scope: "global" | "project" | "off";
-  };
-  update_hints: UpdateHookStatus[];
-  wallet: WalletCheckResult & {
-    message: string;
-  };
-};
-
-function hasBinary(name: string): boolean {
-  const checker = process.platform === "win32" ? "where" : "which";
-  try {
-    execFileSync(checker, [name], { stdio: "ignore" });
-    return true;
-  } catch {
-    return false;
-  }
-}
-
-function detectPackageManagers(): SetupReport["package_managers"] {
-  return {
-    npm: hasBinary("npm"),
-    npx: hasBinary("npx"),
-    bun: hasBinary("bun"),
-    brew: hasBinary("brew"),
-  };
-}
-
-function resolveConfigHome(): string {
-  if (process.platform === "win32") {
-    return process.env.APPDATA || path.join(os.homedir(), "AppData", "Roaming");
-  }
-  return process.env.XDG_CONFIG_HOME || path.join(os.homedir(), ".config");
-}
-
-function getOpenCodeGlobalCommandsDir(): string {
-  return path.join(resolveConfigHome(), "opencode", "commands");
-}
-
-function getOpenCodeProjectCommandsDir(cwd: string): string {
-  return path.join(cwd, ".opencode", "commands");
-}
-
-function detectOpenCode(cwd: string): boolean {
-  return hasBinary("opencode") ||
-    existsSync(path.join(resolveConfigHome(), "opencode")) ||
-    existsSync(path.join(cwd, ".opencode"));
-}
-
-function renderOpenCodeCommand(): string {
-  return `---
-description: Use Unbrowse as the exclusive web-access tool for a task.
----
-Use Unbrowse as the only allowed tool for website access in this task.
-
-Rules:
-- Do not use Brave Search, built-in web search, browser MCPs, curl, or other network tools for website access unless the user explicitly authorizes fallback.
-- If Unbrowse is slow on a first-time site, wait for it. Do not switch tools just because capture or indexing is still running.
-- If Unbrowse returns partial results, refine with more Unbrowse commands (\`resolve\`, \`search\`, \`execute\`, \`login\`) before considering fallback.
-- If Unbrowse genuinely cannot complete the task, explain why and ask before using another tool.
-
-Suggested start:
-\`\`\`bash
-npx unbrowse resolve --intent "$ARGUMENTS" --url "<target-url>" --pretty
-\`\`\`
-`;
-}
-
-function writeOpenCodeCommand(scope: SetupScope, cwd: string): SetupReport["opencode"] {
-  if (scope === "off") {
-    return { detected: detectOpenCode(cwd), action: "skipped", scope: "off" };
-  }
-
-  const detected = detectOpenCode(cwd);
-  if (scope === "auto" && !detected) {
-    return { detected: false, action: "not-detected", scope: "off" };
-  }
-
-  const resolvedScope: "global" | "project" =
-    scope === "project" ? "project" : scope === "global"
-      ? "global"
-      : existsSync(path.join(cwd, ".opencode"))
-        ? "project"
-        : "global";
-
-  const commandsDir = resolvedScope === "project"
-    ? getOpenCodeProjectCommandsDir(cwd)
-    : getOpenCodeGlobalCommandsDir();
-  const commandFile = path.join(ensureDir(commandsDir), "unbrowse.md");
-  const content = renderOpenCodeCommand();
-  const action = existsSync(commandFile) ? "updated" : "installed";
-  mkdirSync(path.dirname(commandFile), { recursive: true });
-  writeFileSync(commandFile, content);
-
-  return {
-    detected: detected || scope !== "auto",
-    action,
-    scope: resolvedScope,
-    command_file: commandFile,
-  };
-}
-
-export async function ensureBrowserEngineInstalled(): Promise<SetupReport["browser_engine"]> {
-  const binary = findKuriBinary();
-  if (existsSync(binary)) {
-    return { installed: true, action: "already-installed" };
-  }
-
-  const sourceDir = getKuriSourceCandidates().find((candidate) => existsSync(path.join(candidate, "build.zig")));
-  if (!sourceDir) {
-    return {
-      installed: false,
-      action: "failed",
-      message: `Kuri binary not found. Checked ${binary}`,
-    };
-  }
-
-  if (!hasBinary("zig")) {
-    return {
-      installed: false,
-      action: "failed",
-      message: `Kuri source found at ${sourceDir}, but Zig is not installed`,
-    };
-  }
-
-  try {
-    execFileSync("zig", ["build", "-Doptimize=ReleaseFast"], {
-      cwd: sourceDir,
-      stdio: "inherit",
-      timeout: 300_000,
-    });
-    const builtBinary = findKuriBinary();
-    if (existsSync(builtBinary)) {
-      return {
-        installed: true,
-        action: "installed",
-        message: `Built Kuri from ${sourceDir}`,
-      };
-    }
-    return {
-      installed: false,
-      action: "failed",
-      message: `Kuri build completed but ${builtBinary} was not created`,
-    };
-  } catch (error) {
-    const message = error instanceof Error ? error.message : String(error);
-    return { installed: false, action: "failed", message };
-  }
-}
-
-export async function runSetup(options?: {
-  cwd?: string;
-  opencode?: SetupScope;
-  installBrowser?: boolean;
-}): Promise<SetupReport> {
-  const cwd = options?.cwd || process.cwd();
-  const installSource = saveInstallSource(import.meta.url);
-
-  const hostEnv = detectHostEnvironment();
-  log("setup", `detected host environment: ${hostEnv}`);
-
-  const browser = options?.installBrowser === false
-    ? { installed: false, action: "skipped" as const }
-    : await ensureBrowserEngineInstalled();
-  const walletCheck = checkWalletConfigured();
-  const skipWalletSetup = process.env.UNBROWSE_SKIP_WALLET_SETUP === "1";
-  const lobsterInstalled = hasBinary("lobstercash") ||
-    existsSync(path.join(os.homedir(), ".agents", "skills", "lobstercash", "SKILL.md"));
-
-  // Auto-setup Crossmint lobster.cash if the wallet tooling is already present.
-  if (!skipWalletSetup && !walletCheck.configured && lobsterInstalled) {
-    console.log("[unbrowse] Crossmint lobster.cash detected but wallet not configured — running wallet setup...");
-    try {
-      execFileSync("npx", ["@crossmint/lobster-cli", "setup"], {
-        stdio: "inherit",
-        timeout: 60_000,
-      });
-      // Re-check after setup
-      const recheck = checkWalletConfigured();
-      if (recheck.configured) {
-        console.log(`[unbrowse] wallet configured (${recheck.provider})`);
-      }
-    } catch {
-      console.warn("[unbrowse] Crossmint lobster.cash setup failed or was skipped — continuing without wallet");
-    }
-  }
-
-  // Re-check wallet state after potential setup
-  const finalWalletCheck = checkWalletConfigured();
-  const wallet = {
-    ...finalWalletCheck,
-    lobster_installed: lobsterInstalled,
-    message: finalWalletCheck.configured
-      ? `Wallet configured (${finalWalletCheck.provider}). This address is the contributor truth: it is synced onto your agent profile, used for contributor payouts when your routes earn, and used for paid-route spending.`
-      : lobsterInstalled
-        ? "Crossmint lobster.cash is installed but not paired. Pair it now so this wallet address becomes your contributor payout target and your paid-route spending wallet. Run: npx @crossmint/lobster-cli setup"
-        : "No wallet configured. Recommended for new installs: set up Crossmint lobster.cash so contributor payouts have a destination address and paid-route spending can clear automatically. Without it you stay in free indexing mode only.",
-    install_hint: finalWalletCheck.configured
-      ? undefined
-      : lobsterInstalled
-        ? "npx @crossmint/lobster-cli setup"
-        : "npx @crossmint/lobster-cli setup",
-  };
-
-  return {
-    os: {
-      platform: process.platform,
-      release: os.release(),
-      arch: process.arch,
-    },
-    host_environment: hostEnv,
-    package_managers: detectPackageManagers(),
-    browser_engine: browser,
-    opencode: writeOpenCodeCommand(options?.opencode ?? "auto", cwd),
-    update_hints: configureUpdateHintHooks(import.meta.url, installSource),
-    wallet,
-  };
-}