unbrowse 1.1.4 → 1.2.0

package/dist/cli.js

@@ -30,25 +30,34 @@ import { randomBytes } from "crypto";
 import { createInterface } from "readline";
 var API_URL = process.env.UNBROWSE_BACKEND_URL || "https://beta-api.unbrowse.ai";
 var PROFILE_NAME = sanitizeProfileName(process.env.UNBROWSE_PROFILE ?? "");
-var CONFIG_DIR = PROFILE_NAME ? join(homedir(), ".unbrowse", "profiles", PROFILE_NAME) : join(homedir(), ".unbrowse");
-var CONFIG_PATH = join(CONFIG_DIR, "config.json");
 var recentLocalSkills = new Map;
 var LOCAL_ONLY = process.env.UNBROWSE_LOCAL_ONLY === "1";
+function getConfigDir() {
+  if (process.env.UNBROWSE_CONFIG_DIR)
+    return process.env.UNBROWSE_CONFIG_DIR;
+  return PROFILE_NAME ? join(homedir(), ".unbrowse", "profiles", PROFILE_NAME) : join(homedir(), ".unbrowse");
+}
+function getConfigPath() {
+  return join(getConfigDir(), "config.json");
+}
 function sanitizeProfileName(value) {
   return value.trim().replace(/[^a-zA-Z0-9._-]+/g, "-").replace(/^-+|-+$/g, "");
 }
 function loadConfig() {
   try {
-    if (existsSync(CONFIG_PATH)) {
-      return JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
+    const configPath = getConfigPath();
+    if (existsSync(configPath)) {
+      return JSON.parse(readFileSync(configPath, "utf-8"));
     }
   } catch {}
   return null;
 }
 function saveConfig(config) {
-  if (!existsSync(CONFIG_DIR))
-    mkdirSync(CONFIG_DIR, { recursive: true });
-  writeFileSync(CONFIG_PATH, JSON.stringify(config, null, 2), { mode: 384 });
+  const configDir = getConfigDir();
+  const configPath = getConfigPath();
+  if (!existsSync(configDir))
+    mkdirSync(configDir, { recursive: true });
+  writeFileSync(configPath, JSON.stringify(config, null, 2), { mode: 384 });
 }
 var EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/i;
 function normalizeAgentEmail(value) {
@@ -77,6 +86,63 @@ function getApiKey() {
   return "";
 }
 var API_TIMEOUT_MS = parseInt(process.env.UNBROWSE_API_TIMEOUT ?? "8000", 10);
+async function validateApiKey(key) {
+  const controller = new AbortController;
+  const timer = setTimeout(() => controller.abort(), API_TIMEOUT_MS);
+  try {
+    const res = await fetch(`${API_URL}/v1/agents/me`, {
+      method: "GET",
+      headers: {
+        "Accept-Encoding": "gzip, deflate",
+        Authorization: `Bearer ${key}`
+      },
+      signal: controller.signal
+    });
+    let detail = "";
+    try {
+      const body = await res.json();
+      detail = body.error ?? body.message ?? "";
+    } catch {}
+    if (res.ok)
+      return { status: "ok" };
+    if (res.status === 404 && /agent profile not found/i.test(detail)) {
+      return { status: "missing_profile", detail };
+    }
+    if (res.status === 401 || res.status === 403) {
+      return { status: "invalid", detail: detail || `HTTP ${res.status}` };
+    }
+    return { status: "offline", detail: detail || `HTTP ${res.status}` };
+  } catch (err) {
+    return { status: "offline", detail: err.message };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function findUsableApiKey() {
+  const envKey = process.env.UNBROWSE_API_KEY?.trim() ?? "";
+  const configKey = loadConfig()?.api_key?.trim() ?? "";
+  if (envKey) {
+    const envStatus = await validateApiKey(envKey);
+    if (envStatus.status === "ok")
+      return { key: envKey, source: "env" };
+    if (envStatus.status === "offline")
+      return { key: envKey, source: "env" };
+    console.warn(`[unbrowse] Ignoring ${envStatus.status === "missing_profile" ? "stale" : "invalid"} UNBROWSE_API_KEY${envStatus.detail ? ` (${envStatus.detail})` : ""}.`);
+  }
+  if (configKey && configKey !== envKey) {
+    const configStatus = await validateApiKey(configKey);
+    if (configStatus.status === "ok") {
+      process.env.UNBROWSE_API_KEY = configKey;
+      return { key: configKey, source: "config" };
+    }
+    if (configStatus.status === "offline") {
+      process.env.UNBROWSE_API_KEY = configKey;
+      return { key: configKey, source: "config" };
+    }
+    console.warn(`[unbrowse] Saved registration is ${configStatus.status === "missing_profile" ? "stale" : "invalid"}${configStatus.detail ? ` (${configStatus.detail})` : ""}. Re-registering.`);
+  }
+  return null;
+}
 async function api(method, path, body, opts) {
   const key = opts?.noAuth ? "" : getApiKey();
   const controller = new AbortController;
@@ -204,7 +270,11 @@ The Unbrowse Terms of Service have been updated.`);
 async function ensureRegistered(options) {
   if (LOCAL_ONLY)
     return;
-  if (getApiKey()) {
+  const usableKey = await findUsableApiKey();
+  if (usableKey) {
+    if (usableKey.source === "config") {
+      console.log("[unbrowse] Restored saved registration.");
+    }
     await checkTosStatus();
     return;
   }
@@ -615,24 +685,28 @@ function buildEntityIndex(items) {
 function detectEntityIndex(data) {
   if (data == null || typeof data !== "object")
     return null;
-  const obj = data;
-  const candidates = [];
-  if (Array.isArray(obj.included))
-    candidates.push(obj.included);
-  if (obj.data && typeof obj.data === "object") {
-    const d = obj.data;
-    if (Array.isArray(d.included))
-      candidates.push(d.included);
-  }
-  for (const arr of candidates) {
+  let best = null;
+  const check = (arr) => {
     if (arr.length < 2)
-      continue;
-    const sample = arr.slice(0, 5);
+      return;
+    const sample = arr.slice(0, 10);
     const withUrn = sample.filter((i) => i != null && typeof i === "object" && typeof i.entityUrn === "string").length;
-    if (withUrn >= sample.length * 0.5)
-      return buildEntityIndex(arr);
+    if (withUrn >= sample.length * 0.5 && (!best || arr.length > best.length)) {
+      best = arr;
+    }
+  };
+  const obj = data;
+  for (const val of Object.values(obj)) {
+    if (Array.isArray(val)) {
+      check(val);
+    } else if (val != null && typeof val === "object" && !Array.isArray(val)) {
+      for (const nested of Object.values(val)) {
+        if (Array.isArray(nested))
+          check(nested);
+      }
+    }
   }
-  return null;
+  return best ? buildEntityIndex(best) : null;
 }
 function resolvePath(obj, path5, entityIndex) {
   if (!path5 || obj == null)
@@ -658,7 +732,7 @@ function resolvePath(obj, path5, entityIndex) {
     }
     const rec = cur;
     let val = rec[seg];
-    if (val === undefined && entityIndex) {
+    if (val == null && entityIndex) {
       const ref = rec[`*${seg}`];
       if (typeof ref === "string") {
         val = entityIndex.get(ref);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "1.1.4",
+  "version": "1.2.0",
   "description": "Reverse-engineer any website into reusable API skills. npm CLI + local engine.",
   "type": "module",
   "bin": {

package/runtime-src/api/routes.ts

@@ -32,8 +32,8 @@ async function fetchStats() {
       .then(r => r.json() as Promise<{ downloads?: number }>);
 
   const npmRange = (pkg: string) =>
+    fetch(`https://api.npmjs.org/downloads/range/last-month/${pkg}`)
       .then(r => r.json() as Promise<{ downloads?: Array<{ day: string; downloads: number }> }>);
-      .then(r => r.json() as Promise<{ downloads?: number; downloads?: Array<{ day: string; downloads: number }> }>);
 
   const externalCalls: Promise<unknown>[] = [
     npmPoint("unbrowse", "last-month"),

package/runtime-src/capture/index.ts

@@ -20,6 +20,7 @@ const activeBrowserRegistry = new Set<InstanceType<typeof BrowserManager>>();
 // Hard timeout per capture: 90s prevents stuck browsers from holding slots forever
 const CAPTURE_TIMEOUT_MS = 90_000;
 const BROWSER_CLOSE_TIMEOUT_MS = 4_000;
+const CAPTURE_NAV_TIMEOUT_MS = 20_000;
 
 async function closeBrowserSafely(browser: InstanceType<typeof BrowserManager>): Promise<void> {
   await Promise.race([
@@ -28,6 +29,26 @@ async function closeBrowserSafely(browser: InstanceType<typeof BrowserManager>):
   ]);
 }
 
+type CaptureNavigationPage = {
+  goto(url: string, options: { waitUntil: "domcontentloaded"; timeout: number }): Promise<unknown>;
+};
+
+export async function navigatePageForCapture(page: CaptureNavigationPage, url: string): Promise<void> {
+  await page.goto(url, { waitUntil: "domcontentloaded", timeout: CAPTURE_NAV_TIMEOUT_MS });
+}
+
+async function navigateBrowserForCapture(
+  browser: InstanceType<typeof BrowserManager>,
+  url: string,
+): Promise<void> {
+  try {
+    await navigatePageForCapture(browser.getPage(), url);
+    return;
+  } catch {
+    await executeCommand({ action: "navigate", id: nanoid(), url }, browser);
+  }
+}
+
 async function acquireBrowserSlot(): Promise<void> {
   if (activeBrowsers < MAX_CONCURRENT_BROWSERS) {
     activeBrowsers++;
@@ -80,6 +101,44 @@ export interface RawRequest {
   timestamp: string;
 }
 
+export type CapturedCookie = {
+  name: string;
+  value: string;
+  domain: string;
+  path?: string;
+  httpOnly?: boolean;
+  secure?: boolean;
+};
+
+export function filterFirstPartySessionCookies(
+  cookies: CapturedCookie[],
+  ...urls: Array<string | undefined>
+): CapturedCookie[] {
+  const hosts = new Set<string>();
+  const domains = new Set<string>();
+  for (const rawUrl of urls) {
+    if (!rawUrl) continue;
+    try {
+      const host = new URL(rawUrl).hostname.toLowerCase();
+      hosts.add(host);
+      domains.add(getRegistrableDomain(host));
+    } catch {
+      // ignore bad urls
+    }
+  }
+  if (hosts.size === 0 && domains.size === 0) return cookies;
+  return cookies.filter((cookie) => {
+    const cookieDomain = cookie.domain.replace(/^\./, "").toLowerCase();
+    if (!cookieDomain) return false;
+    if (hosts.has(cookieDomain)) return true;
+    try {
+      return domains.has(getRegistrableDomain(cookieDomain));
+    } catch {
+      return false;
+    }
+  });
+}
+
 export function isBlockedAppShell(html?: string): boolean {
   if (!html) return false;
   return (
@@ -526,7 +585,7 @@ export async function captureSession(
     // CDP session unavailable — skip WS capture
   }
 
-  await executeCommand({ action: "navigate", id: nanoid(), url }, browser);
+  await navigateBrowserForCapture(browser, url);
 
   // Adaptive wait: handle Cloudflare challenges + SPA content loading + intent-aware API wait
   await waitForContentReady(browser, url, intent, responseBodies);
@@ -599,14 +658,18 @@ export async function captureSession(
   // Extract session cookies so callers can persist auth for future executions
   const ctx = browser.getContext();
   const rawCookies = ctx ? await ctx.cookies().catch(() => []) : [];
-  const sessionCookies = rawCookies.map((c) => ({
-    name: c.name,
-    value: c.value,
-    domain: c.domain,
-    path: c.path,
-    httpOnly: c.httpOnly,
-    secure: c.secure,
-  }));
+  const sessionCookies = filterFirstPartySessionCookies(
+    rawCookies.map((c) => ({
+      name: c.name,
+      value: c.value,
+      domain: c.domain,
+      path: c.path,
+      httpOnly: c.httpOnly,
+      secure: c.secure,
+    })),
+    url,
+    final_url,
+  );
 
   if (captureTimedOut) throw new Error(`captureSession timed out after ${CAPTURE_TIMEOUT_MS}ms for ${url}`);
   log("capture", `captured ${jsBundleBodies.size} JS bundles for route scanning`);
@@ -667,7 +730,7 @@ export async function executeInBrowser(
   // No auth: use in-page fetch (original path)
   browser.startRequestTracking();
   const origin = new URL(url).origin;
-  await executeCommand({ action: "navigate", id: nanoid(), url: origin }, browser);
+  await navigateBrowserForCapture(browser, origin);
 
   const page = browser.getPage();
   const result = await page.evaluate(

package/runtime-src/cli.ts

@@ -118,28 +118,38 @@ function buildEntityIndex(items: unknown[]): Map<string, unknown> {
   return index;
 }
 
-/** Detect if an object contains a normalized entity array and build the index. */
+/** Detect if an object contains a normalized entity array and build the index.
+ *  Searches all top-level and one-level-nested arrays for entityUrn-keyed items,
+ *  picking the largest qualifying array. Works for any normalized API shape. */
 function detectEntityIndex(data: unknown): Map<string, unknown> | null {
   if (data == null || typeof data !== "object") return null;
-  const obj = data as Record<string, unknown>;
 
-  // Check common locations: { included: [...] }, { data: { included: [...] } }
-  const candidates: unknown[][] = [];
-  if (Array.isArray(obj.included)) candidates.push(obj.included);
-  if (obj.data && typeof obj.data === "object") {
-    const d = obj.data as Record<string, unknown>;
-    if (Array.isArray(d.included)) candidates.push(d.included);
-  }
+  let best: unknown[] | null = null;
 
-  for (const arr of candidates) {
-    if (arr.length < 2) continue;
-    const sample = arr.slice(0, 5);
+  const check = (arr: unknown[]) => {
+    if (arr.length < 2) return;
+    const sample = arr.slice(0, 10);
     const withUrn = sample.filter(
       (i) => i != null && typeof i === "object" && typeof (i as Record<string, unknown>).entityUrn === "string"
     ).length;
-    if (withUrn >= sample.length * 0.5) return buildEntityIndex(arr);
+    if (withUrn >= sample.length * 0.5 && (!best || arr.length > best.length)) {
+      best = arr;
+    }
+  };
+
+  const obj = data as Record<string, unknown>;
+  for (const val of Object.values(obj)) {
+    if (Array.isArray(val)) {
+      check(val);
+    } else if (val != null && typeof val === "object" && !Array.isArray(val)) {
+      // One level deep: { data: { included: [...] } }, { response: { entities: [...] } }, etc.
+      for (const nested of Object.values(val as Record<string, unknown>)) {
+        if (Array.isArray(nested)) check(nested);
+      }
+    }
   }
-  return null;
+
+  return best ? buildEntityIndex(best) : null;
 }
 
 /** Resolve a dot-path like "data.items[].name" against an object.
@@ -165,8 +175,10 @@ function resolvePath(obj: unknown, path: string, entityIndex?: Map<string, unkno
     const rec = cur as Record<string, unknown>;
     let val = rec[seg];
 
-    // URN reference resolution: if direct lookup fails, check for "*key" reference
-    if (val === undefined && entityIndex) {
+    // URN reference resolution: if direct lookup fails (or is null), check for "*key" reference.
+    // Normalized APIs (LinkedIn Voyager, Facebook Graph) set inline fields to null when
+    // the value is stored as a URN reference: e.g. socialDetail: null + *socialDetail: "urn:li:..."
+    if (val == null && entityIndex) {
       const ref = rec[`*${seg}`];
       if (typeof ref === "string") {
         val = entityIndex.get(ref);

package/runtime-src/client/index.ts

@@ -7,10 +7,6 @@ import type { AgentSkillChunkView, EndpointStats, ExecutionTrace, OrchestrationT
 
 const API_URL = process.env.UNBROWSE_BACKEND_URL || "https://beta-api.unbrowse.ai";
 const PROFILE_NAME = sanitizeProfileName(process.env.UNBROWSE_PROFILE ?? "");
-const CONFIG_DIR = PROFILE_NAME
-  ? join(homedir(), ".unbrowse", "profiles", PROFILE_NAME)
-  : join(homedir(), ".unbrowse");
-const CONFIG_PATH = join(CONFIG_DIR, "config.json");
 const recentLocalSkills = new Map<string, SkillManifest>();
 const LOCAL_ONLY = process.env.UNBROWSE_LOCAL_ONLY === "1";
 
@@ -19,7 +15,18 @@ function scopedSkillKey(skillId: string, scopeId?: string): string {
 }
 
 function getSkillCacheDir(): string {
-  return process.env.UNBROWSE_SKILL_CACHE_DIR || join(CONFIG_DIR, "skill-cache");
+  return process.env.UNBROWSE_SKILL_CACHE_DIR || join(getConfigDir(), "skill-cache");
+}
+
+function getConfigDir(): string {
+  if (process.env.UNBROWSE_CONFIG_DIR) return process.env.UNBROWSE_CONFIG_DIR;
+  return PROFILE_NAME
+    ? join(homedir(), ".unbrowse", "profiles", PROFILE_NAME)
+    : join(homedir(), ".unbrowse");
+}
+
+function getConfigPath(): string {
+  return join(getConfigDir(), "config.json");
 }
 
 function sanitizeProfileName(value: string): string {
@@ -43,18 +50,29 @@ interface UnbrowseConfig {
   tos_accepted_at: string | null;
 }
 
+type ApiKeySource = "env" | "config";
+type ApiKeyValidationStatus = "ok" | "missing_profile" | "invalid" | "offline";
+
+interface ApiKeyValidationResult {
+  status: ApiKeyValidationStatus;
+  detail?: string;
+}
+
 function loadConfig(): UnbrowseConfig | null {
   try {
-    if (existsSync(CONFIG_PATH)) {
-      return JSON.parse(readFileSync(CONFIG_PATH, "utf-8"));
+    const configPath = getConfigPath();
+    if (existsSync(configPath)) {
+      return JSON.parse(readFileSync(configPath, "utf-8"));
     }
   } catch { /* corrupt file, re-register */ }
   return null;
 }
 
 function saveConfig(config: UnbrowseConfig): void {
-  if (!existsSync(CONFIG_DIR)) mkdirSync(CONFIG_DIR, { recursive: true });
-  writeFileSync(CONFIG_PATH, JSON.stringify(config, null, 2), { mode: 0o600 });
+  const configDir = getConfigDir();
+  const configPath = getConfigPath();
+  if (!existsSync(configDir)) mkdirSync(configDir, { recursive: true });
+  writeFileSync(configPath, JSON.stringify(config, null, 2), { mode: 0o600 });
 }
 
 const EMAIL_RE = /^[^\s@]+@[^\s@]+\.[^\s@]+$/i;
@@ -90,6 +108,67 @@ export function getApiKey(): string {
 
 const API_TIMEOUT_MS = parseInt(process.env.UNBROWSE_API_TIMEOUT ?? "8000", 10);
 
+async function validateApiKey(key: string): Promise<ApiKeyValidationResult> {
+  const controller = new AbortController();
+  const timer = setTimeout(() => controller.abort(), API_TIMEOUT_MS);
+  try {
+    const res = await fetch(`${API_URL}/v1/agents/me`, {
+      method: "GET",
+      headers: {
+        "Accept-Encoding": "gzip, deflate",
+        Authorization: `Bearer ${key}`,
+      },
+      signal: controller.signal,
+    });
+
+    let detail = "";
+    try {
+      const body = await res.json() as { error?: string; message?: string };
+      detail = body.error ?? body.message ?? "";
+    } catch {}
+
+    if (res.ok) return { status: "ok" };
+    if (res.status === 404 && /agent profile not found/i.test(detail)) {
+      return { status: "missing_profile", detail };
+    }
+    if (res.status === 401 || res.status === 403) {
+      return { status: "invalid", detail: detail || `HTTP ${res.status}` };
+    }
+    return { status: "offline", detail: detail || `HTTP ${res.status}` };
+  } catch (err) {
+    return { status: "offline", detail: (err as Error).message };
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function findUsableApiKey(): Promise<{ key: string; source: ApiKeySource } | null> {
+  const envKey = process.env.UNBROWSE_API_KEY?.trim() ?? "";
+  const configKey = loadConfig()?.api_key?.trim() ?? "";
+
+  if (envKey) {
+    const envStatus = await validateApiKey(envKey);
+    if (envStatus.status === "ok") return { key: envKey, source: "env" };
+    if (envStatus.status === "offline") return { key: envKey, source: "env" };
+    console.warn(`[unbrowse] Ignoring ${envStatus.status === "missing_profile" ? "stale" : "invalid"} UNBROWSE_API_KEY${envStatus.detail ? ` (${envStatus.detail})` : ""}.`);
+  }
+
+  if (configKey && configKey !== envKey) {
+    const configStatus = await validateApiKey(configKey);
+    if (configStatus.status === "ok") {
+      process.env.UNBROWSE_API_KEY = configKey;
+      return { key: configKey, source: "config" };
+    }
+    if (configStatus.status === "offline") {
+      process.env.UNBROWSE_API_KEY = configKey;
+      return { key: configKey, source: "config" };
+    }
+    console.warn(`[unbrowse] Saved registration is ${configStatus.status === "missing_profile" ? "stale" : "invalid"}${configStatus.detail ? ` (${configStatus.detail})` : ""}. Re-registering.`);
+  }
+
+  return null;
+}
+
 async function api<T = unknown>(method: string, path: string, body?: unknown, opts?: { noAuth?: boolean }): Promise<T> {
   const key = opts?.noAuth ? "" : getApiKey();
   const controller = new AbortController();
@@ -239,8 +318,11 @@ async function checkTosStatus(): Promise<void> {
 /** Auto-register with the backend if no API key is configured. Persists to ~/.unbrowse/config.json. */
 export async function ensureRegistered(options?: { promptForEmail?: boolean }): Promise<void> {
   if (LOCAL_ONLY) return;
-  if (getApiKey()) {
-    // Already have a key — check if ToS re-acceptance is needed
+  const usableKey = await findUsableApiKey();
+  if (usableKey) {
+    if (usableKey.source === "config") {
+      console.log("[unbrowse] Restored saved registration.");
+    }
     await checkTosStatus();
     return;
   }
@@ -520,6 +602,39 @@ export async function searchIntentInDomain(
   return data.results;
 }
 
+export async function searchIntentResolve(
+  intent: string,
+  domain?: string,
+  domainK = 5,
+  globalK = 10,
+): Promise<{
+  domain_results: Array<{ id: number; score: number; metadata: Record<string, unknown> }>;
+  global_results: Array<{ id: number; score: number; metadata: Record<string, unknown> }>;
+  skipped_global: boolean;
+}> {
+  if (LOCAL_ONLY) return { domain_results: [], global_results: [], skipped_global: false };
+  try {
+    return await api<{
+      domain_results: Array<{ id: number; score: number; metadata: Record<string, unknown> }>;
+      global_results: Array<{ id: number; score: number; metadata: Record<string, unknown> }>;
+      skipped_global: boolean;
+    }>("POST", "/v1/search/resolve", {
+      intent,
+      domain,
+      domain_k: domainK,
+      global_k: globalK,
+    });
+  } catch {
+    const [domain_results, global_results] = await Promise.all([
+      domain
+        ? searchIntentInDomain(intent, domain, domainK).catch(() => [] as Array<{ id: number; score: number; metadata: Record<string, unknown> }>)
+        : Promise.resolve([] as Array<{ id: number; score: number; metadata: Record<string, unknown> }>),
+      searchIntent(intent, globalK).catch(() => [] as Array<{ id: number; score: number; metadata: Record<string, unknown> }>),
+    ]);
+    return { domain_results, global_results, skipped_global: false };
+  }
+}
+
 // --- Stats ---
 
 export async function recordExecution(