unbrowse 3.0.2 → 3.1.0-experiments.5e7a7bb

package/runtime-src/client/index.ts

@@ -497,7 +497,8 @@ async function apiRequest<T = unknown>(
     throw new Error("ToS update required. Restart unbrowse to accept new terms.");
   }
 
-  // Handle x402 payment required — surface payment terms to the caller
+
+  // Handle x402 payment required — attempt lobster pay-and-retry before surfacing
   if (res.status === 402) {
     const paymentRequired = res.headers.get("PAYMENT-REQUIRED");
     const legacyPaymentTerms = res.headers.get("X-Payment-Required");
@@ -506,6 +507,29 @@ async function apiRequest<T = unknown>(
       : legacyPaymentTerms
         ? JSON.parse(legacyPaymentTerms)
         : (data as Record<string, unknown>).terms;
+
+    // Try lobster.cash automatic payment before throwing
+    try {
+      const { isLobsterAvailable, payAndRetry } = await import("../payments/lobster-pay.js");
+      if (isLobsterAvailable()) {
+        const fullUrl = `${API_URL}${path}`;
+        const paidResult = await payAndRetry<T>(fullUrl, {
+          body,
+          headers: {
+            "Content-Type": "application/json",
+            "Accept-Encoding": "gzip, deflate",
+            ...releaseAttestationHeaders,
+            ...(key ? { Authorization: `Bearer ${key}` } : {}),
+          },
+        });
+        if (paidResult) {
+          return { data: paidResult.data, headers: new Headers() };
+        }
+      }
+    } catch (payErr) {
+      console.warn(`[x402] lobster pay-and-retry failed: ${(payErr as Error).message}`);
+    }
+
     const err = new Error(`Payment required: ${(data as Record<string, unknown>).error ?? "This skill requires payment"}`);
     (err as Error & { x402: boolean; terms: unknown; status: number }).x402 = true;
     (err as Error & { terms: unknown }).terms = terms;
@@ -528,18 +552,10 @@ async function api<T = unknown>(method: string, path: string, body?: unknown, op
 
 // --- Install attribution ---
 
-function parseInstallAttribution(): { install_attribution?: Record<string, string>; landing_token?: string } {
-  const result: { install_attribution?: Record<string, string>; landing_token?: string } = {};
-  const b64 = process.env.UNBROWSE_ATTRIBUTION_B64;
-  if (b64) {
-    try {
-      const decoded = JSON.parse(Buffer.from(b64, "base64").toString("utf8"));
-      if (decoded && typeof decoded === "object") result.install_attribution = decoded;
-    } catch { /* malformed — ignore */ }
-  }
+function parseInstallAttribution(): { landing_token?: string } {
   const token = process.env.UNBROWSE_LANDING_TOKEN;
-  if (token && token.length < 2048) result.landing_token = token;
-  return result;
+  if (token && token.length < 2048) return { landing_token: token };
+  return {};
 }
 
 // --- ToS acceptance ---

package/runtime-src/execution/index.ts

@@ -1882,28 +1882,40 @@ export async function executeEndpoint(
       wallet_configured: !!wallet.wallet_address,
     });
     if (gate.status === "payment_required" || gate.status === "wallet_not_configured" || gate.status === "insufficient_balance") {
-      const trace: ExecutionTrace = stampTrace({
-        trace_id: nanoid(),
-        skill_id: skill.skill_id,
-        endpoint_id: endpoint.endpoint_id,
-        started_at: new Date().toISOString(),
-        completed_at: new Date().toISOString(),
-        success: false,
-        status_code: 402,
-        error: "payment_required",
-      });
-      return {
-        trace,
-        result: {
+      // If lobster wallet is available, let execution proceed —
+      // the client-level apiRequest will handle 402 pay-and-retry automatically.
+      let lobsterAvailable = false;
+      try {
+        const { isLobsterAvailable } = await import("../payments/lobster-pay.js");
+        lobsterAvailable = isLobsterAvailable();
+      } catch {}
+
+      if (lobsterAvailable && gate.status === "payment_required") {
+        console.log(`[payment] ${skill.skill_id}: lobster available — proceeding with auto-pay`);
+      } else {
+        const trace: ExecutionTrace = stampTrace({
+          trace_id: nanoid(),
+          skill_id: skill.skill_id,
+          endpoint_id: endpoint.endpoint_id,
+          started_at: new Date().toISOString(),
+          completed_at: new Date().toISOString(),
+          success: false,
+          status_code: 402,
           error: "payment_required",
-          price_usd: gate.requirement?.amount,
-          payment_status: gate.status,
-          message: gate.message,
-          wallet_provider: wallet.wallet_provider ?? "lobster.cash",
-          wallet_address: wallet.wallet_address,
-          indexing_fallback_available: true,
-        },
-      };
+        });
+        return {
+          trace,
+          result: {
+            error: "payment_required",
+            price_usd: gate.requirement?.amount,
+            payment_status: gate.status,
+            message: gate.message,
+            wallet_provider: wallet.wallet_provider ?? "lobster.cash",
+            wallet_address: wallet.wallet_address,
+            indexing_fallback_available: true,
+          },
+        };
+      }
     }
   }
 
@@ -2005,6 +2017,16 @@ export async function executeEndpoint(
   const epDomain = (() => { try { return new URL(endpoint.url_template).hostname; } catch { return skill.domain; } })();
   await reloadExecutionAuthState(skill, epDomain, authHeaders, cookies);
 
+  // If endpoint has auth_tokens bindings and vault didn't provide the needed headers,
+  // resolve them from the page source (meta tags, inline scripts, JS bundles)
+  if (endpoint.auth_tokens?.length && Object.keys(authHeaders).length === 0) {
+    try {
+      const { resolveAuthTokens } = await import("./token-resolver.js");
+      const resolved = await resolveAuthTokens(endpoint, cookies, authHeaders);
+      Object.assign(authHeaders, resolved);
+    } catch { /* token resolution is best-effort */ }
+  }
+
   log("exec", `endpoint ${endpoint.endpoint_id}: cookies=${cookies.length} authHeaders=${Object.keys(authHeaders).length} hasAuth=${cookies.length > 0 || Object.keys(authHeaders).length > 0}`);
 
   // BUG-006: Merge path_params defaults — user params override captured defaults

package/runtime-src/execution/token-resolver.ts

@@ -0,0 +1,122 @@
+/**
+ * Token resolver — resolves auth_tokens bindings at execute time.
+ *
+ * Given an endpoint with auth_tokens, loads the trigger page via Kuri,
+ * extracts fresh token values from HTML/JS sources, and returns headers
+ * ready to inject into the outgoing request.
+ *
+ * This closes the gap for sites that rotate CSRF tokens per page-load,
+ * keep bearer tokens in JS bundles, or hydrate tokens into inline scripts.
+ */
+
+import type { EndpointDescriptor, AuthTokenBinding } from "../types/index.js";
+import { extractTokenFromHtml, extractTokenFromBundle } from "../reverse-engineer/token-sources.js";
+import * as kuri from "../kuri/client.js";
+
+const RESOLVE_TIMEOUT_MS = 12000;
+
+/**
+ * Resolve auth_tokens bindings by loading the trigger page and scraping
+ * token values from their known source locations.
+ *
+ * Returns a map of header-name → resolved-value for all successfully
+ * resolved header-type bindings. Returns empty map if no bindings or
+ * all fail.
+ */
+export async function resolveAuthTokens(
+  endpoint: EndpointDescriptor,
+  cookies: Array<{ name: string; value: string; domain: string }>,
+  existingAuthHeaders: Record<string, string>,
+): Promise<Record<string, string>> {
+  const bindings = endpoint.auth_tokens;
+  if (!bindings || bindings.length === 0) return {};
+
+  const triggerUrl = endpoint.trigger_url;
+  if (!triggerUrl) return {};
+
+  // Only resolve header bindings that aren't already satisfied
+  const headerBindings = bindings.filter(
+    (b) => b.param_location === "header" && !existingAuthHeaders[b.param_name],
+  );
+  if (headerBindings.length === 0) return {};
+
+  const resolved: Record<string, string> = {};
+
+  try {
+    // Open a tab, inject cookies, navigate to trigger page
+    const tabId = await openResolverTab(triggerUrl, cookies);
+    if (!tabId) return {};
+
+    try {
+      await waitForLoad(tabId);
+
+      const html = await kuri.getPageHtml(tabId).catch(() => "");
+      if (typeof html !== "string" || !html.startsWith("<")) {
+        return {};
+      }
+
+      for (const binding of headerBindings) {
+        const value = resolveBinding(binding, html);
+        if (value) {
+          resolved[binding.param_name] = binding.param_name.toLowerCase() === "authorization"
+            ? (value.startsWith("Bearer ") ? value : `Bearer ${value}`)
+            : value;
+        }
+      }
+    } finally {
+      await kuri.closeTab(tabId).catch(() => {});
+    }
+  } catch {
+    // Tab open/nav failed — return whatever we resolved so far
+  }
+
+  return resolved;
+}
+
+function resolveBinding(binding: AuthTokenBinding, html: string): string | undefined {
+  for (const source of binding.sources) {
+    let value: string | undefined;
+
+    if (source.kind === "html-meta" || source.kind === "html-inline-script") {
+      value = extractTokenFromHtml(source, html);
+    }
+    // JS bundle resolution would require fetching the bundle URL —
+    // skip for now, HTML sources cover most cases (CSRF, inline tokens)
+
+    if (value && value.length >= 8) return value;
+  }
+  return undefined;
+}
+
+async function openResolverTab(
+  url: string,
+  cookies: Array<{ name: string; value: string; domain: string }>,
+): Promise<string | undefined> {
+  try {
+    const tab = await kuri.newTab(url);
+    const tabId = typeof tab === "string" ? tab : (tab as { tab_id?: string })?.tab_id;
+    if (!tabId) return undefined;
+
+    if (cookies.length > 0) {
+      for (const c of cookies) {
+        await kuri.setCookie(tabId, c.name, c.value, c.domain).catch(() => {});
+      }
+      await kuri.navigate(tabId, url).catch(() => {});
+    }
+
+    return tabId;
+  } catch {
+    return undefined;
+  }
+}
+
+async function waitForLoad(tabId: string): Promise<void> {
+  const start = Date.now();
+  while (Date.now() - start < RESOLVE_TIMEOUT_MS) {
+    try {
+      const state = await kuri.evaluate(tabId, "document.readyState");
+      if (state === "complete" || state === "interactive") return;
+    } catch { /* page not ready */ }
+    await new Promise((r) => setTimeout(r, 500));
+  }
+}

package/runtime-src/graph/index.ts

@@ -373,15 +373,20 @@ export function getEndpointDescriptionMetadata(endpoint: Pick<EndpointDescriptor
   warning?: string;
 } {
   const input = classifyDescriptionInput(endpoint.description);
-  const display = (input.source === "agent"
-    ? endpoint.description ?? ""
+  // Trust the LLM augmenter's semantic layer when it has run.
+  const agentAugmented = endpoint.semantic?.description_source === "agent";
+  // Schema-grounded auto descriptions (with real response fields) don't need
+  // external LLM review — the calling agent IS the LLM that reviews them.
+  const schemaGrounded = !!(endpoint.semantic?.example_fields?.length || endpoint.semantic?.response_summary);
+  const display = (input.source === "agent" || agentAugmented
+    ? endpoint.semantic?.description_out ?? endpoint.description ?? ""
     : endpoint.semantic?.description_out ?? endpoint.description ?? "").trim();
-  const source = display ? (input.source === "agent" ? "agent" : "auto") : "missing";
+  const source = display ? (input.source === "agent" || agentAugmented ? "agent" : "auto") : "missing";
   return {
     display,
     source,
-    needs_review: source !== "agent",
-    ...(source !== "agent"
+    needs_review: source === "missing" || (source === "auto" && !schemaGrounded),
+    ...(source !== "agent" && !schemaGrounded
       ? { warning: input.warning ?? "Auto-generated description. Review before trusting or publishing." }
       : {}),
   };
@@ -931,9 +936,12 @@ export function inferEndpointSemantic(
     confidence: fields.length > 0 ? 0.8 : 0.4,
     observed_at: opts?.observedAt,
     sample_request_url: opts?.sampleRequestUrl,
+    auth_required: !!(
+      endpoint.auth_tokens?.length ||
+      Object.keys(endpoint.headers_template ?? {}).some((h) => /auth|csrf|token|bearer/i.test(h))
+    ),
   };
 }
-
 export function resolveEndpointSemantic(
   endpoint: EndpointDescriptor,
   opts?: {

package/runtime-src/impact-log.ts

@@ -0,0 +1,227 @@
+/**
+ * Local per-agent impact log.
+ *
+ * Every resolve/execute appends one line here with the savings it delivered
+ * (time, tokens, dollars, browser avoided). `unbrowse stats` and the
+ * unbrowse_stats MCP tool read this file to show lifetime impact without
+ * needing a new backend endpoint.
+ *
+ * Format: JSONL at ~/.unbrowse/impact-log.jsonl (or $UNBROWSE_CONFIG_DIR).
+ * Rotates when the file exceeds ~5MB.
+ */
+
+import { existsSync, mkdirSync, appendFileSync, statSync, readFileSync, renameSync, unlinkSync } from "node:fs";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
+
+const MAX_LOG_BYTES = 5 * 1024 * 1024;
+const MAX_ROTATIONS = 3;
+
+export interface ImpactLogEntry {
+  ts: string;
+  command: "resolve" | "execute" | "browse" | string;
+  source?: string;
+  domain?: string;
+  intent?: string;
+  skill_id?: string;
+  endpoint_id?: string;
+  time_saved_ms?: number;
+  time_saved_pct?: number;
+  tokens_saved?: number;
+  tokens_saved_pct?: number;
+  cost_saved_uc?: number;
+  browser_avoided?: boolean;
+  success?: boolean;
+}
+
+export interface ImpactSummary {
+  total_runs: number;
+  successful_runs: number;
+  browser_avoided_runs: number;
+  total_time_saved_ms: number;
+  total_tokens_saved: number;
+  total_cost_saved_uc: number;
+  avg_time_saved_pct: number;
+  avg_tokens_saved_pct: number;
+  by_source: Record<string, number>;
+  first_entry_at: string | null;
+  last_entry_at: string | null;
+}
+
+function getLogDir(): string {
+  if (process.env.UNBROWSE_CONFIG_DIR) return process.env.UNBROWSE_CONFIG_DIR;
+  const profile = process.env.UNBROWSE_PROFILE?.trim();
+  return profile
+    ? join(homedir(), ".unbrowse", "profiles", profile)
+    : join(homedir(), ".unbrowse");
+}
+
+export function getImpactLogPath(): string {
+  return join(getLogDir(), "impact-log.jsonl");
+}
+
+function ensureDir(path: string): void {
+  const dir = dirname(path);
+  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+}
+
+function rotateIfNeeded(path: string): void {
+  try {
+    if (!existsSync(path)) return;
+    const size = statSync(path).size;
+    if (size < MAX_LOG_BYTES) return;
+    for (let i = MAX_ROTATIONS; i >= 1; i--) {
+      const older = `${path}.${i}`;
+      if (!existsSync(older)) continue;
+      if (i === MAX_ROTATIONS) {
+        try { unlinkSync(older); } catch {}
+      } else {
+        try { renameSync(older, `${path}.${i + 1}`); } catch {}
+      }
+    }
+    renameSync(path, `${path}.1`);
+  } catch {
+    // best-effort; never throw from logging
+  }
+}
+
+/** Append a single impact record. Fire-and-forget; never throws. */
+export function appendImpact(entry: ImpactLogEntry): void {
+  try {
+    const hasSignal =
+      (entry.time_saved_ms ?? 0) > 0 ||
+      (entry.tokens_saved ?? 0) > 0 ||
+      (entry.cost_saved_uc ?? 0) > 0 ||
+      entry.browser_avoided === true;
+    if (!hasSignal) return;
+
+    const path = getImpactLogPath();
+    ensureDir(path);
+    rotateIfNeeded(path);
+    appendFileSync(path, JSON.stringify(entry) + "\n", "utf8");
+  } catch {
+    // never surface logging errors
+  }
+}
+
+/**
+ * Extract an ImpactLogEntry from an orchestrator result's `impact` field.
+ * Returns null if the result has no impact data.
+ */
+export function impactFromResult(
+  command: string,
+  result: unknown,
+  extras: { intent?: string; domain?: string; skill_id?: string; endpoint_id?: string } = {},
+): ImpactLogEntry | null {
+  if (!result || typeof result !== "object") return null;
+  const r = result as Record<string, unknown>;
+  const impact = (r.impact ?? null) as Record<string, unknown> | null;
+  if (!impact || typeof impact !== "object") return null;
+
+  const num = (v: unknown): number | undefined =>
+    typeof v === "number" && Number.isFinite(v) ? v : undefined;
+
+  return {
+    ts: new Date().toISOString(),
+    command,
+    source: typeof impact.source === "string" ? impact.source : undefined,
+    domain: extras.domain,
+    intent: extras.intent,
+    skill_id: extras.skill_id ?? (typeof r.skill_id === "string" ? r.skill_id : undefined),
+    endpoint_id: extras.endpoint_id ?? (typeof r.endpoint_id === "string" ? r.endpoint_id : undefined),
+    time_saved_ms: num(impact.time_saved_ms),
+    time_saved_pct: num(impact.time_saved_pct),
+    tokens_saved: num(impact.tokens_saved),
+    tokens_saved_pct: num(impact.tokens_saved_pct),
+    cost_saved_uc: num(impact.cost_saved_uc),
+    browser_avoided: impact.browser_avoided === true,
+    success: r.error == null,
+  };
+}
+
+/** Read and aggregate the full impact log (across rotations). Safe on missing file. */
+export function readImpactSummary(): ImpactSummary {
+  const path = getImpactLogPath();
+  const summary: ImpactSummary = {
+    total_runs: 0,
+    successful_runs: 0,
+    browser_avoided_runs: 0,
+    total_time_saved_ms: 0,
+    total_tokens_saved: 0,
+    total_cost_saved_uc: 0,
+    avg_time_saved_pct: 0,
+    avg_tokens_saved_pct: 0,
+    by_source: {},
+    first_entry_at: null,
+    last_entry_at: null,
+  };
+
+  const files: string[] = [];
+  for (let i = MAX_ROTATIONS; i >= 1; i--) {
+    const rotated = `${path}.${i}`;
+    if (existsSync(rotated)) files.push(rotated);
+  }
+  if (existsSync(path)) files.push(path);
+  if (files.length === 0) return summary;
+
+  let timePctSum = 0;
+  let timePctCount = 0;
+  let tokenPctSum = 0;
+  let tokenPctCount = 0;
+
+  for (const file of files) {
+    let raw: string;
+    try {
+      raw = readFileSync(file, "utf8");
+    } catch {
+      continue;
+    }
+    for (const line of raw.split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed) continue;
+      let e: ImpactLogEntry;
+      try {
+        e = JSON.parse(trimmed) as ImpactLogEntry;
+      } catch {
+        continue;
+      }
+      summary.total_runs += 1;
+      if (e.success !== false) summary.successful_runs += 1;
+      if (e.browser_avoided) summary.browser_avoided_runs += 1;
+      summary.total_time_saved_ms += e.time_saved_ms ?? 0;
+      summary.total_tokens_saved += e.tokens_saved ?? 0;
+      summary.total_cost_saved_uc += e.cost_saved_uc ?? 0;
+      if (typeof e.time_saved_pct === "number") {
+        timePctSum += e.time_saved_pct;
+        timePctCount += 1;
+      }
+      if (typeof e.tokens_saved_pct === "number") {
+        tokenPctSum += e.tokens_saved_pct;
+        tokenPctCount += 1;
+      }
+      if (e.source) {
+        summary.by_source[e.source] = (summary.by_source[e.source] ?? 0) + 1;
+      }
+      if (!summary.first_entry_at || e.ts < summary.first_entry_at) summary.first_entry_at = e.ts;
+      if (!summary.last_entry_at || e.ts > summary.last_entry_at) summary.last_entry_at = e.ts;
+    }
+  }
+
+  summary.avg_time_saved_pct = timePctCount > 0 ? Math.round(timePctSum / timePctCount) : 0;
+  summary.avg_tokens_saved_pct = tokenPctCount > 0 ? Math.round(tokenPctSum / tokenPctCount) : 0;
+  return summary;
+}
+
+/** For tests only. */
+export function _clearImpactLogForTests(): void {
+  const path = getImpactLogPath();
+  try {
+    if (existsSync(path)) unlinkSync(path);
+    for (let i = 1; i <= MAX_ROTATIONS + 1; i++) {
+      const rotated = `${path}.${i}`;
+      if (existsSync(rotated)) unlinkSync(rotated);
+    }
+  } catch {
+    // ignore
+  }
+}

package/runtime-src/kuri/client.ts

@@ -216,7 +216,10 @@ function falseyEnv(value: string | undefined): boolean {
 }
 
 export function resolveKuriLaunchConfig(env: NodeJS.ProcessEnv = process.env): KuriLaunchConfig {
-  const headless = envFlag(env.KURI_HEADLESS ?? env.HEADLESS);
+  const explicitHeadless = env.KURI_HEADLESS ?? env.HEADLESS;
+  const headless = explicitHeadless !== undefined
+    ? envFlag(explicitHeadless)
+    : (process.platform === "linux" && !env.DISPLAY); // auto-headless when no display on Linux
   const cleanRoom = envFlag(env.UNBROWSE_LOCAL_ONLY) || envFlag(env.KURI_CLEAN_ROOM);
   const browserCookieOptOut = falseyEnv(env.UNBROWSE_IMPORT_BROWSER_COOKIES);
   const explicitAttach = envFlag(env.KURI_ATTACH_EXISTING_CHROME ?? env.UNBROWSE_ATTACH_EXISTING_CHROME);
@@ -240,6 +243,7 @@ function currentBundledKuriTarget(): string | null {
   if (process.platform === "darwin" && process.arch === "x64") return "darwin-x64";
   if (process.platform === "linux" && process.arch === "arm64") return "linux-arm64";
   if (process.platform === "linux" && process.arch === "x64") return "linux-x64";
+  if (process.platform === "win32" && process.arch === "x64") return "win-x64";
   return null;
 }
 

package/runtime-src/marketplace/index.ts

@@ -82,11 +82,19 @@ export function mergeEndpoints(
       dom_extraction: ep.dom_extraction ?? dupe.dom_extraction,
       semantic: ep.semantic ?? dupe.semantic,
       response_schema: ep.response_schema ?? dupe.response_schema,
-      headers_template: ep.headers_template ?? dupe.headers_template,
+      headers_template: Object.keys(ep.headers_template ?? {}).length > 0 ? ep.headers_template : dupe.headers_template,
       query: ep.query ?? dupe.query,
       path_params: ep.path_params ?? dupe.path_params,
       body: ep.body ?? dupe.body,
+      body_params: ep.body_params ?? dupe.body_params,
       trigger_url: ep.trigger_url ?? dupe.trigger_url,
+      csrf_plan: ep.csrf_plan ?? dupe.csrf_plan,
+      oauth_plan: ep.oauth_plan ?? dupe.oauth_plan,
+      search_form: ep.search_form ?? dupe.search_form,
+      policy: ep.policy ?? dupe.policy,
+      graph_visibility: ep.graph_visibility ?? dupe.graph_visibility,
+      corroboration: ep.corroboration ?? dupe.corroboration,
+      auth_tokens: ep.auth_tokens ?? dupe.auth_tokens,
     };
   }
   return merged;