unbrowse 2.0.21 → 2.0.23

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "2.0.21",
+  "version": "2.0.23",
   "description": "Reverse-engineer any website into reusable API skills. npm CLI + local engine.",
   "type": "module",
   "bin": {

package/runtime-src/auth/profile-context.ts

@@ -38,6 +38,25 @@ export interface LaunchedProfileContext {
   tempDir: string;
 }
 
+async function waitForChildExit(child: ChildProcess, timeoutMs = 2_000): Promise<void> {
+  if (child.exitCode !== null || child.killed) return;
+  await new Promise<void>((resolve) => {
+    const timer = setTimeout(resolve, timeoutMs);
+    child.once("exit", () => {
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+}
+
+function removeTempDirQuietly(dir: string): void {
+  try {
+    rmSync(dir, { recursive: true, force: true });
+  } catch {
+    // best-effort cleanup; do not fail captures on temp profile removal
+  }
+}
+
 function resolveChromiumBinary(browserName: string): string | null {
   const macos = new Map<string, string>([
     ["Chrome", "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome"],
@@ -296,13 +315,14 @@ export async function launchChromiumProfileContext(meta: BrowserAuthSourceMeta):
     };
   } catch (error) {
     try { child.kill("SIGTERM"); } catch {}
-    rmSync(tempDir, { recursive: true, force: true });
+    removeTempDirQuietly(tempDir);
     throw error;
   }
 }
 
-export function cleanupProfileContext(ctx: LaunchedProfileContext | null | undefined): void {
+export async function cleanupProfileContext(ctx: LaunchedProfileContext | null | undefined): Promise<void> {
   if (!ctx) return;
   try { ctx.child.kill("SIGTERM"); } catch {}
-  rmSync(ctx.tempDir, { recursive: true, force: true });
+  await waitForChildExit(ctx.child);
+  removeTempDirQuietly(ctx.tempDir);
 }

package/runtime-src/capture/index.ts

@@ -268,6 +268,20 @@ export function blockedAppShellErrorCode(
   return hasAuth ? "blocked_app_shell" : "auth_required";
 }
 
+export function shouldShortCircuitEmbeddedPayloadCapture(url: string, intent: string | undefined, html?: string): boolean {
+  if (!html) return false;
+  const lowerIntent = intent?.toLowerCase() ?? "";
+  if (
+    /linkedin\.com/i.test(url) &&
+    /\/feed(?:\/|$)/i.test(url) &&
+    /\b(feed|timeline|stream|post|posts|update|updates|home)\b/.test(lowerIntent) &&
+    /voyagerFeedDashMainFeed/.test(html)
+  ) {
+    return true;
+  }
+  return false;
+}
+
 function shouldRetryEphemeralProfileError(error: unknown): boolean {
   const message = error instanceof Error ? error.message : String(error ?? "");
   return /persistentcontext|target page, context or browser has been closed|browser has been closed|page has been closed/i.test(message);
@@ -930,16 +944,23 @@ export async function captureSession(
       }
       await kuri.stop();
       kuri.useExternalChrome(browserCdpBaseUrl(profileCtx.cdpUrl), { child: profileCtx.child, tempDir: profileCtx.tempDir });
+      let nestedResult: CaptureResult | null = null;
       try {
-        return await captureSession(url, undefined, undefined, intent, {
+        nestedResult = await captureSession(url, undefined, undefined, intent, {
           ...options,
           forceEphemeral: true,
           usedProfileContext: true,
           preferExistingTab,
           authStrategy: "header-replay",
         });
+        return nestedResult;
       } finally {
-        await kuri.stop();
+        try {
+          await kuri.stop();
+        } catch (stopErr) {
+          log("capture", `profile-context cleanup failed for ${url}: ${stopErr instanceof Error ? stopErr.message : String(stopErr)}`);
+          if (!nestedResult) throw stopErr;
+        }
       }
     } catch (attachErr) {
       log("capture", `forced profile context failed for ${url}: ${attachErr instanceof Error ? attachErr.message : String(attachErr)}`);
@@ -1081,6 +1102,40 @@ export async function captureSession(
       await kuri.evaluate(tabId, INTERCEPTOR_SCRIPT);
     } catch { /* page may not be ready */ }
 
+    // For pages that embed the task payload directly in the HTML, return before
+    // the longer network/intercept wait. This avoids losing useful captures to
+    // later browser-engine instability on auth-gated SPAs like LinkedIn feed.
+    try {
+      await sleep(1_500, signal);
+      throwIfAborted(signal);
+      const earlyHtml = await kuri.getPageHtml(tabId);
+      if (shouldShortCircuitEmbeddedPayloadCapture(url, intent, earlyHtml)) {
+        let final_url = url;
+        try {
+          const rawUrl = await kuri.getCurrentUrl(tabId);
+          final_url = typeof rawUrl === "string" ? rawUrl : String(rawUrl ?? url);
+          try { new URL(final_url); } catch { final_url = url; }
+        } catch {
+          final_url = url;
+        }
+        lastHtml = earlyHtml;
+        const rawCookies = await extractCookiesFromPage(tabId, url);
+        const sessionCookies = filterFirstPartySessionCookies(rawCookies, url, final_url);
+        log("capture", `short-circuiting embedded payload capture for ${url}`);
+        return {
+          requests: [],
+          har_lineage_id: nanoid(),
+          domain,
+          cookies: sessionCookies,
+          final_url,
+          html: earlyHtml,
+          js_bundles: new Map(),
+        };
+      }
+    } catch {
+      // fall through to the longer capture path
+    }
+
     // Build response bodies map from intercepted requests
     const responseBodies = new Map<string, string>();
     const jsBundleBodies = new Map<string, string>();

package/runtime-src/execution/index.ts

@@ -27,7 +27,7 @@ import { buildSkillOperationGraph, inferEndpointSemantic, resolveEndpointSemanti
 import { augmentEndpointsWithAgent } from "../graph/agent-augment.js";
 import { log } from "../logger.js";
 import { TRACE_VERSION } from "../version.js";
-import { buildQueryBindingMap, extractTemplateQueryBindings, mergeContextTemplateParams } from "../template-params.js";
+import { buildQueryBindingMap, buildTemplatedQuery, extractTemplateQueryBindings, extractTemplateVariables, mergeContextTemplateParams, parseStructuredQueryTuple } from "../template-params.js";
 import { assessIntentResult, projectIntentData } from "../intent-match.js";
 import * as cheerio from "cheerio";
 
@@ -782,59 +782,93 @@ function buildLinkedInEmbeddedFeedCapture(
     return {};
   }
 
-  try {
-    const $ = cheerio.load(html);
-    let metadata: {
-      request?: string;
-      method?: string;
-      headers?: Record<string, string>;
-      body?: string;
-    } | null = null;
-
-    $("code").each((_, el) => {
-      if (metadata) return;
-      const text = $(el).text().trim();
-      if (!/voyagerFeedDashMainFeed/.test(text)) return;
-      if (!/"request":"\/voyager\/api\/graphql/.test(text)) return;
+  const $ = cheerio.load(html);
+  let metadata: {
+    request?: string;
+    method?: string;
+    headers?: Record<string, string>;
+    body?: string;
+  } | null = null;
+
+  $("code").each((_, el) => {
+    if (metadata) return;
+    const text = $(el).text().trim();
+    if (!/voyagerFeedDashMainFeed/.test(text)) return;
+    if (!/"request":"\/voyager\/api\/graphql/.test(text)) return;
+    try {
       metadata = JSON.parse(text);
-    });
-    if (!metadata?.body) return {};
-
-    let payloadText = "";
-    $("code").each((_, el) => {
-      if (payloadText) return;
-      const id = $(el).attr("id");
-      if (id !== metadata?.body) return;
-      payloadText = $(el).text().trim();
-    });
-    if (!payloadText) return {};
-
-    const payload = JSON.parse(payloadText);
-    const semanticAssessment = assessIntentResult(payload, intent);
-    if (semanticAssessment.verdict === "fail") {
-      return { quality_note: semanticAssessment.reason };
+    } catch {
+      metadata = null;
     }
+  });
+  if (!metadata?.body) return {};
+
+  let payloadText = "";
+  $("code").each((_, el) => {
+    if (payloadText) return;
+    const id = $(el).attr("id");
+    if (id !== metadata?.body) return;
+    payloadText = $(el).text().trim();
+  });
+  if (!payloadText) return {};
 
-    const requestUrl = metadata.request?.startsWith("http")
-      ? metadata.request
-      : `https://www.linkedin.com${metadata.request?.startsWith("/") ? "" : "/"}${metadata.request ?? ""}`;
-    if (!requestUrl || requestUrl === "https://www.linkedin.com/") return {};
+  let payload: unknown;
+  try {
+    payload = JSON.parse(payloadText);
+  } catch {
+    return {};
+  }
 
-    const endpoint: EndpointDescriptor = {
-      endpoint_id: nanoid(),
-      method: (metadata.method ?? "GET").toUpperCase() as EndpointDescriptor["method"],
-      url_template: requestUrl,
-      exec_strategy: "trigger-intercept",
-      idempotency: "safe",
-      verification_status: "verified",
-      reliability_score: 0.95,
-      description: `Embedded LinkedIn feed payload for ${intent}`,
-      response_schema: inferSchema([payload]),
-      trigger_url: url,
-      ...(metadata.headers && Object.keys(metadata.headers).length > 0
-        ? { headers_template: metadata.headers }
-        : {}),
-    };
+  const semanticAssessment = assessIntentResult(payload, intent);
+  if (semanticAssessment.verdict === "fail") {
+    return { quality_note: semanticAssessment.reason };
+  }
+
+  const requestUrl = metadata.request?.startsWith("http")
+    ? metadata.request
+    : `https://www.linkedin.com${metadata.request?.startsWith("/") ? "" : "/"}${metadata.request ?? ""}`;
+  if (!requestUrl || requestUrl === "https://www.linkedin.com/") return {};
+
+  const queryDefaults = (() => {
+    try {
+      return Object.fromEntries(new URL(requestUrl).searchParams.entries());
+    } catch {
+      return {} as Record<string, string>;
+    }
+  })();
+  let urlTemplate = requestUrl;
+  try {
+    const parsed = new URL(requestUrl);
+    const templatedQuery = buildTemplatedQuery(queryDefaults);
+    const query = Object.entries(templatedQuery)
+      .map(([key, value]) => `${encodeURIComponent(key)}=${value}`)
+      .join("&");
+    urlTemplate = query ? `${parsed.origin}${parsed.pathname}?${query}` : `${parsed.origin}${parsed.pathname}`;
+  } catch {
+    urlTemplate = requestUrl;
+  }
+
+  const endpoint: EndpointDescriptor = {
+    endpoint_id: nanoid(),
+    method: (metadata.method ?? "GET").toUpperCase() as EndpointDescriptor["method"],
+    url_template: urlTemplate,
+    exec_strategy: "trigger-intercept",
+    idempotency: "safe",
+    verification_status: "verified",
+    reliability_score: 0.95,
+    description: `Embedded LinkedIn feed payload for ${intent}`,
+    trigger_url: url,
+    ...(Object.keys(queryDefaults).length > 0 ? { query: queryDefaults } : {}),
+    ...(metadata.headers && Object.keys(metadata.headers).length > 0
+      ? { headers_template: metadata.headers }
+      : {}),
+  };
+  try {
+    endpoint.response_schema = inferSchema([payload]);
+  } catch {
+    // keep embedded endpoint even if schema inference chokes on the payload
+  }
+  try {
     endpoint.semantic = {
       ...inferEndpointSemantic(endpoint, {
         sampleResponse: payload,
@@ -844,21 +878,21 @@ function buildLinkedInEmbeddedFeedCapture(
       }),
       ...(authRequired ? { auth_required: true } : {}),
     };
-
-    return {
-      endpoint,
-      result: {
-        data: payload,
-        _extraction: {
-          method: "linkedin-embedded-feed",
-          confidence: 0.95,
-          source: "html-embedded",
-        },
-      },
-    };
   } catch {
-    return {};
+    endpoint.semantic = authRequired ? { action_kind: "timeline", resource_kind: "post", auth_required: true } : undefined;
   }
+
+  return {
+    endpoint,
+    result: {
+      data: payload,
+      _extraction: {
+        method: "linkedin-embedded-feed",
+        confidence: 0.95,
+        source: "html-embedded",
+      },
+    },
+  };
 }
 
 export function buildPageArtifactCapture(
@@ -2380,6 +2414,7 @@ export async function executeEndpoint(
       }
     }
   }
+  applyStructuredQueryDefaults(mergedParams, endpoint.url_template, endpoint.query);
 
   // Merge captured query params into URL — user params override endpoint defaults
   let urlTemplate = resolveExecutionUrlTemplate(endpoint, options?.contextUrl);
@@ -2388,12 +2423,23 @@ export async function executeEndpoint(
       const u = new URL(urlTemplate);
       const queryBindings = extractTemplateQueryBindings(endpoint.url_template);
       for (const [k, v] of Object.entries(endpoint.query)) {
+        const currentTemplateValue = u.searchParams.get(k) ?? "";
+        const structuredOverride = typeof v === "string"
+          ? mergeStructuredQueryValue(currentTemplateValue, v, mergedParams)
+          : null;
+        const hasStructuredPlaceholders = parseStructuredQueryTuple(currentTemplateValue)?.some((entry) =>
+          extractTemplateVariables(entry.value).length > 0
+        ) ?? false;
         const bindingKey = queryBindings[k];
         // User params override captured query defaults
         if (bindingKey && mergedParams[bindingKey] != null) {
           u.searchParams.set(k, String(mergedParams[bindingKey]));
         } else if (mergedParams[k] != null) {
           u.searchParams.set(k, String(mergedParams[k]));
+        } else if (structuredOverride) {
+          u.searchParams.set(k, structuredOverride);
+        } else if (hasStructuredPlaceholders) {
+          continue;
         } else if (v != null) {
           u.searchParams.set(k, String(v));
         }
@@ -2416,6 +2462,13 @@ export async function executeEndpoint(
       ...Object.keys(endpoint.path_params ?? {}),
       ...Object.keys(endpoint.query ?? {}),
     ]);
+    for (const value of Object.values(endpoint.query ?? {})) {
+      if (typeof value !== "string") continue;
+      for (const entry of parseStructuredQueryTuple(value) ?? []) {
+        consumedKeys.add(entry.key);
+        for (const placeholder of extractTemplateVariables(entry.value)) consumedKeys.add(placeholder);
+      }
+    }
     for (const [rawKey, bindingKey] of Object.entries(extractTemplateQueryBindings(endpoint.url_template))) {
       consumedKeys.add(rawKey);
       consumedKeys.add(bindingKey);
@@ -2891,6 +2944,68 @@ function interpolate(template: string, params: Record<string, unknown>): string
   return `${interpolatedBase}?${interpolatedQuery}`;
 }
 
+function applyStructuredQueryDefaults(
+  mergedParams: Record<string, unknown>,
+  urlTemplate: string,
+  queryDefaults?: Record<string, unknown>,
+): void {
+  if (!queryDefaults || Object.keys(queryDefaults).length === 0) return;
+  try {
+    const templateUrl = new URL(urlTemplate);
+    for (const [key, rawValue] of Object.entries(queryDefaults)) {
+      if (typeof rawValue !== "string") continue;
+      const templateValue = templateUrl.searchParams.get(key);
+      if (!templateValue) continue;
+      const templateTuple = parseStructuredQueryTuple(templateValue);
+      const defaultTuple = parseStructuredQueryTuple(rawValue);
+      if (!templateTuple || !defaultTuple || templateTuple.length === 0 || defaultTuple.length === 0) continue;
+      const defaultByKey = new Map(defaultTuple.map((entry) => [entry.key, entry.value]));
+      for (const entry of templateTuple) {
+        const placeholder = entry.value.match(/^\{([^}]+)\}$/)?.[1];
+        if (!placeholder || mergedParams[placeholder] != null) continue;
+        const fallback = defaultByKey.get(entry.key);
+        if (fallback != null && fallback !== "") mergedParams[placeholder] = fallback;
+      }
+    }
+  } catch {
+    // ignore malformed template URL
+  }
+}
+
+function mergeStructuredQueryValue(
+  currentValue: string,
+  fallbackValue: string | undefined,
+  mergedParams: Record<string, unknown>,
+): string | null {
+  const templateTuple = parseStructuredQueryTuple(currentValue);
+  const fallbackTuple = fallbackValue ? parseStructuredQueryTuple(fallbackValue) : null;
+  const activeTuple = templateTuple ?? fallbackTuple;
+  if (!activeTuple || activeTuple.length === 0) return null;
+
+  const fallbackByKey = new Map((fallbackTuple ?? []).map((entry) => [entry.key, entry.value]));
+  let changed = false;
+  const rewritten = activeTuple.map((entry) => {
+    const placeholder = entry.value.match(/^\{([^}]+)\}$/)?.[1];
+    const directOverride = mergedParams[entry.key];
+    const placeholderOverride = placeholder ? mergedParams[placeholder] : undefined;
+    const nextValue = placeholderOverride ?? directOverride;
+    if (nextValue != null) {
+      changed = true;
+      return `${entry.key}:${String(nextValue)}`;
+    }
+    if (placeholder) {
+      const fallback = fallbackByKey.get(entry.key);
+      if (fallback != null && fallback !== "") {
+        changed = true;
+        return `${entry.key}:${fallback}`;
+      }
+    }
+    return `${entry.key}:${entry.value}`;
+  });
+
+  return changed ? `(${rewritten.join(",")})` : null;
+}
+
 function interpolateObj(
   obj: Record<string, unknown>,
   params: Record<string, unknown>

package/runtime-src/kuri/client.ts

@@ -65,6 +65,18 @@ let externalChromeOverride: {
   previousAttach?: string;
 } | null = null;
 
+async function waitForChildExit(child: ChildProcess | null | undefined, timeoutMs = 2_000): Promise<void> {
+  if (!child) return;
+  if (child.exitCode !== null || child.killed) return;
+  await new Promise<void>((resolve) => {
+    const timer = setTimeout(resolve, timeoutMs);
+    child.once("exit", () => {
+      clearTimeout(timer);
+      resolve();
+    });
+  });
+}
+
 function kuriBinaryName(): string {
   return process.platform === "win32" ? "kuri.exe" : "kuri";
 }
@@ -426,7 +438,12 @@ export async function stop(): Promise<void> {
       // ignore
     }
     if (externalChromeOverride.tempDir) {
-      rmSync(externalChromeOverride.tempDir, { recursive: true, force: true });
+      await waitForChildExit(externalChromeOverride.child);
+      try {
+        rmSync(externalChromeOverride.tempDir, { recursive: true, force: true });
+      } catch {
+        // best-effort cleanup; don't fail the caller on temp profile removal
+      }
     }
     if (externalChromeOverride.previousCdpUrl == null) delete process.env.CDP_URL;
     else process.env.CDP_URL = externalChromeOverride.previousCdpUrl;