unbrowse 3.0.2 → 3.1.0-experiments.5e7a7bb

package/runtime-src/payments/lobster-pay.ts

@@ -0,0 +1,182 @@
+/**
+ * Lobster.cash x402 payment bridge.
+ *
+ * Handles the pay-and-retry cycle:
+ *   1. Receives x402 payment terms from a 402 response
+ *   2. Calls `lobstercash x402 fetch <url>` to pay + retry
+ *   3. Returns the paid response body
+ *
+ * Delegation boundary:
+ * - Unbrowse owns: detecting 402, passing the URL, using the result
+ * - Lobster owns: wallet signing, transaction broadcast, proof construction
+ */
+
+import { execFile, execFileSync } from "node:child_process";
+import { existsSync } from "node:fs";
+import { homedir } from "node:os";
+import { join } from "node:path";
+
+const LOBSTER_PAY_TIMEOUT_MS = 30_000;
+
+/**
+ * Resolve the lobster CLI command. Prefers the direct binary if
+ * installed globally, otherwise falls back to npx.
+ */
+function getLobsterCommand(): { cmd: string; prefix: string[] } | null {
+  // 1. Direct binary in PATH
+  try {
+    execFileSync("lobstercash", ["--version"], { stdio: "ignore", timeout: 3_000 });
+    return { cmd: "lobstercash", prefix: [] };
+  } catch (_e) { /* not in PATH */ }
+
+  // 2. Check npm global bin (often not in agent PATH)
+  try {
+    const npmPrefix = execFileSync("npm", ["config", "get", "prefix"], { encoding: "utf8", timeout: 5_000 }).trim();
+    const lobsterPath = join(npmPrefix, "bin", "lobstercash");
+    if (existsSync(lobsterPath)) {
+      execFileSync(lobsterPath, ["--version"], { stdio: "ignore", timeout: 3_000 });
+      return { cmd: lobsterPath, prefix: [] };
+    }
+  } catch (_e) { /* npm not available */ }
+
+  return null;
+}
+
+let cachedCommand: { cmd: string; prefix: string[] } | null | undefined = undefined;
+function lobsterCmd(): { cmd: string; prefix: string[] } | null {
+  if (cachedCommand === undefined) cachedCommand = getLobsterCommand();
+  return cachedCommand;
+}
+
+export interface LobsterPayResult {
+  success: boolean;
+  body: string;
+  statusCode?: number;
+  error?: string;
+}
+
+/**
+ * Check if the lobster CLI is available and wallet is configured.
+ */
+export function isLobsterAvailable(): boolean {
+  const agentsPath = join(process.env.HOME || homedir(), ".lobster", "agents.json");
+  return existsSync(agentsPath);
+}
+
+/**
+ * Pay for an x402-gated URL via lobster.cash and return the response.
+ *
+ * Shells out to `lobstercash x402 fetch <url>` which handles:
+ * - Reading 402 + payment terms
+ * - Signing + broadcasting the USDC transfer
+ * - Retrying with PAYMENT-SIGNATURE header
+ * - Returning the paid response
+ */
+export function lobsterX402Fetch(
+  url: string,
+  options?: {
+    jsonBody?: string;
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+  },
+): Promise<LobsterPayResult> {
+  return new Promise((resolve) => {
+    const resolved = lobsterCmd();
+    if (!resolved) {
+      resolve({ success: false, body: "", error: "lobstercash CLI not in PATH" });
+      return;
+    }
+    const { cmd, prefix } = resolved;
+    const args = [...prefix, "x402", "fetch", url, "--debug"];
+
+    if (options?.jsonBody) {
+      args.push("--json", options.jsonBody);
+    }
+
+    if (options?.headers) {
+      for (const [key, value] of Object.entries(options.headers)) {
+        args.push("--header", `${key}:${value}`);
+      }
+    }
+
+    const timeout = options?.timeoutMs ?? LOBSTER_PAY_TIMEOUT_MS;
+    args.push("--timeout", String(timeout));
+
+    execFile(cmd, args, { timeout: timeout + 5_000, maxBuffer: 1024 * 1024 }, (err, stdout, stderr) => {
+      if (err) {
+        const msg = stderr?.trim() || err.message;
+        console.warn(`[lobster-pay] x402 fetch failed: ${msg}`);
+        resolve({ success: false, body: "", error: msg });
+        return;
+      }
+
+      if (stderr) {
+        for (const line of stderr.split("\n").filter(Boolean)) {
+          console.log(`[lobster-pay] ${line}`);
+        }
+      }
+
+      // Parse status from stdout header line: "Status: 200"
+      const statusMatch = stdout.match(/^Status:\s*(\d+)/m);
+      const statusCode = statusMatch ? parseInt(statusMatch[1], 10) : undefined;
+
+      if (statusCode && statusCode >= 400) {
+        resolve({ success: false, body: stdout, statusCode, error: `HTTP ${statusCode}` });
+        return;
+      }
+
+      resolve({ success: true, body: stdout, statusCode });
+    });
+  });
+}
+
+/**
+ * Pay for an x402-gated API request and return the parsed JSON response.
+ *
+ * This is the high-level entry point for the pay-and-retry loop.
+ * Returns null if payment fails or lobster is unavailable.
+ */
+export async function payAndRetry<T = unknown>(
+  fullUrl: string,
+  options?: {
+    body?: unknown;
+    headers?: Record<string, string>;
+  },
+): Promise<{ data: T; paid: true } | null> {
+  if (!isLobsterAvailable()) {
+    console.log("[lobster-pay] lobster.cash not configured — skipping payment");
+    return null;
+  }
+
+  console.log(`[lobster-pay] attempting x402 payment for ${fullUrl}`);
+
+  const result = await lobsterX402Fetch(fullUrl, {
+    jsonBody: options?.body ? JSON.stringify(options.body) : undefined,
+    headers: options?.headers,
+  });
+
+  if (!result.success) {
+    console.warn(`[lobster-pay] payment failed: ${result.error}`);
+    return null;
+  }
+  try {
+    // lobstercash x402 fetch outputs header lines before the JSON body:
+    //   x402 FETCH <url>
+    //   Status: 200
+    //   Content-Type: application/json
+    //   <blank line>
+    //   {"actual":"json",...}
+    // Extract the JSON portion by finding the first '{' or '['
+    const raw = result.body;
+    const jsonStart = Math.min(
+      ...[raw.indexOf("{"), raw.indexOf("[")].filter((i) => i >= 0),
+    );
+    const jsonStr = jsonStart >= 0 ? raw.slice(jsonStart) : raw;
+    const data = JSON.parse(jsonStr) as T;
+    console.log("[lobster-pay] payment successful — got paid response");
+    return { data, paid: true };
+  } catch (_e) {
+    console.warn("[lobster-pay] paid response was not valid JSON");
+    return null;
+  }
+}

package/runtime-src/reverse-engineer/token-sources.ts

@@ -0,0 +1,357 @@
+/**
+ * Token source discovery - finds where a captured token value lives in HTML / JS
+ * so the value can be re-scraped fresh at replay time.
+ *
+ * Input:  a concrete token value we saw in a captured request header
+ * Output: an ordered list of AuthTokenSource locators (meta tag, inline script regex,
+ *         or JS bundle regex) that will re-produce the same value when re-evaluated
+ *
+ * The resolver (src/execution/token-resolver.ts) walks these sources in order at
+ * replay time and injects the freshest value into the outgoing request. This closes
+ * the gap for sites that rotate CSRF tokens per page-load, keep bearer tokens in JS
+ * bundles (Twitter public API), or hydrate tokens into window.__INITIAL_STATE__.
+ */
+
+import type { AuthTokenSource } from "../types/index.js";
+
+// Minimum token length to consider - below this, false-positive matches dominate
+// (common 8-char hex substrings appear in every page).
+const MIN_TOKEN_LENGTH = 16;
+
+// Maximum sources to return per token - higher values make resolution slower
+// without meaningfully improving robustness.
+const MAX_SOURCES = 5;
+
+/**
+ * Find all deterministic locators that currently produce the given token value
+ * from the supplied HTML and JS bundle contents. The locators are captured in a
+ * form that allows re-evaluation at replay time against freshly-fetched page
+ * state - if the site rotates the token each page-load, the same locator will
+ * return the new value.
+ */
+export function findTokenSources(
+  tokenValue: string,
+  html: string | undefined,
+  jsBundles?: Map<string, string>,
+): AuthTokenSource[] {
+  if (!tokenValue || tokenValue.length < MIN_TOKEN_LENGTH) return [];
+
+  const sources: AuthTokenSource[] = [];
+
+  if (html) {
+    sources.push(...findHtmlMetaSources(tokenValue, html));
+    sources.push(...findInlineScriptSources(tokenValue, html));
+  }
+
+  if (jsBundles && jsBundles.size > 0) {
+    sources.push(...findJsBundleSources(tokenValue, jsBundles));
+  }
+
+  // Dedupe and cap
+  const seen = new Set<string>();
+  const out: AuthTokenSource[] = [];
+  for (const src of sources) {
+    const key = JSON.stringify(src);
+    if (seen.has(key)) continue;
+    seen.add(key);
+    out.push(src);
+    if (out.length >= MAX_SOURCES) break;
+  }
+  return out;
+}
+
+// HTML <meta> scanner
+// Matches patterns like:
+//   <meta name="csrf-token" content="ABC123...">
+//   <meta name="csrfToken" value="ABC123..." />
+//   <meta property="csrf" content="ABC123...">
+
+const META_TAG_PATTERN = /<meta\s+([^>]+?)\s*\/?>/gi;
+
+function findHtmlMetaSources(tokenValue: string, html: string): AuthTokenSource[] {
+  const out: AuthTokenSource[] = [];
+  META_TAG_PATTERN.lastIndex = 0;
+  let match: RegExpExecArray | null;
+  while ((match = META_TAG_PATTERN.exec(html)) !== null) {
+    const attrs = match[1];
+    if (!attrs.includes(tokenValue)) continue;
+
+    const name =
+      extractAttr(attrs, "name") ??
+      extractAttr(attrs, "property") ??
+      extractAttr(attrs, "itemprop");
+    if (!name) continue;
+
+    for (const attrName of ["content", "value"] as const) {
+      const attrValue = extractAttr(attrs, attrName);
+      if (attrValue === tokenValue) {
+        out.push({
+          kind: "html-meta",
+          meta_name: name,
+          meta_attr: attrName,
+        });
+        break;
+      }
+    }
+  }
+  return out;
+}
+
+function extractAttr(attrs: string, name: string): string | undefined {
+  const re = new RegExp(`\\b${escapeRegExp(name)}\\s*=\\s*(?:"([^"]*)"|'([^']*)'|([^\\s>]+))`, "i");
+  const m = attrs.match(re);
+  if (!m) return undefined;
+  return m[1] ?? m[2] ?? m[3];
+}
+
+// Inline <script> scanner
+
+const INLINE_SCRIPT_PATTERN = /<script(?:\s[^>]*)?>([\s\S]*?)<\/script>/gi;
+const CONTEXT_WINDOW = 48;
+
+function findInlineScriptSources(tokenValue: string, html: string): AuthTokenSource[] {
+  const out: AuthTokenSource[] = [];
+  INLINE_SCRIPT_PATTERN.lastIndex = 0;
+  let match: RegExpExecArray | null;
+  while ((match = INLINE_SCRIPT_PATTERN.exec(html)) !== null) {
+    const scriptBody = match[1];
+    const idx = scriptBody.indexOf(tokenValue);
+    if (idx < 0) continue;
+
+    const regex = buildContextRegex(scriptBody, idx, tokenValue);
+    if (!regex) continue;
+
+    out.push({
+      kind: "html-inline-script",
+      inline_script_regex: regex,
+    });
+  }
+  return out;
+}
+
+// JS bundle scanner
+
+function findJsBundleSources(
+  tokenValue: string,
+  bundles: Map<string, string>,
+): AuthTokenSource[] {
+  const out: AuthTokenSource[] = [];
+  for (const [bundleUrl, content] of bundles) {
+    const idx = content.indexOf(tokenValue);
+    if (idx < 0) continue;
+
+    const regex = buildContextRegex(content, idx, tokenValue);
+    if (!regex) continue;
+
+    out.push({
+      kind: "js-bundle",
+      bundle_url_pattern: bundleUrlPattern(bundleUrl),
+      bundle_regex: regex,
+    });
+  }
+  return out;
+}
+
+function bundleUrlPattern(bundleUrl: string): string {
+  try {
+    const u = new URL(bundleUrl);
+    const path = u.pathname.replace(/[.-][a-f0-9]{6,}\./, ".");
+    return path.replace(/\.(m?js)$/i, "");
+  } catch {
+    return bundleUrl.replace(/[.-][a-f0-9]{6,}\./, ".").replace(/\.(m?js)$/i, "");
+  }
+}
+
+// Context regex builder
+
+function buildContextRegex(
+  content: string,
+  tokenIdx: number,
+  tokenValue: string,
+): string | null {
+  const start = Math.max(0, tokenIdx - CONTEXT_WINDOW);
+  let context = content.slice(start, tokenIdx);
+
+  const delimMatch = context.match(/([{,\[(]\s*["'`]?[\w$-]+\s*["'`]?\s*[:=]\s*["'`])[^{,\[(]*$/);
+  if (delimMatch) {
+    context = delimMatch[1];
+  } else {
+    context = context.trimStart();
+    if (context.length < 6) return null;
+  }
+
+  const afterIdx = tokenIdx + tokenValue.length;
+  const terminator = content.charAt(afterIdx);
+
+  let terminatorPattern: string;
+  if (terminator === '"') terminatorPattern = '"';
+  else if (terminator === "'") terminatorPattern = "'";
+  else if (terminator === "`") terminatorPattern = "`";
+  else if (terminator === "," || terminator === "}" || terminator === "]" || terminator === ")") {
+    terminatorPattern = `[${escapeRegExp(terminator)}]`;
+  } else {
+    return null;
+  }
+
+  const tokenCharset = inferTokenCharset(tokenValue);
+  return `${escapeRegExp(context)}(${tokenCharset}+?)(?=${terminatorPattern})`;
+}
+
+function inferTokenCharset(tokenValue: string): string {
+  if (/^[A-Za-z0-9+/=]+$/.test(tokenValue)) return "[A-Za-z0-9+/=]";
+  if (/^[A-Za-z0-9._-]+$/.test(tokenValue)) return "[A-Za-z0-9._-]";
+  if (/^[A-Fa-f0-9]+$/.test(tokenValue)) return "[A-Fa-f0-9]";
+  if (/^[A-Za-z0-9]+$/.test(tokenValue)) return "[A-Za-z0-9]";
+  return "[^\"'`,}\\]\\s)]";
+}
+
+function escapeRegExp(s: string): string {
+  return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+
+// Resolver-side helpers
+
+export function extractTokenFromHtml(
+  source: AuthTokenSource,
+  html: string,
+): string | undefined {
+  if (source.kind === "html-meta" && source.meta_name) {
+    const attr = source.meta_attr ?? "content";
+    const re = new RegExp(
+      `<meta\\s+[^>]*?(?:name|property|itemprop)\\s*=\\s*["']${escapeRegExp(source.meta_name)}["'][^>]*?${attr}\\s*=\\s*["']([^"']+)["'][^>]*>`,
+      "i",
+    );
+    const m = html.match(re);
+    if (m) return m[1];
+    const re2 = new RegExp(
+      `<meta\\s+[^>]*?${attr}\\s*=\\s*["']([^"']+)["'][^>]*?(?:name|property|itemprop)\\s*=\\s*["']${escapeRegExp(source.meta_name)}["'][^>]*>`,
+      "i",
+    );
+    const m2 = html.match(re2);
+    if (m2) return m2[1];
+    return undefined;
+  }
+
+  if (source.kind === "html-inline-script" && source.inline_script_regex) {
+    const scriptPattern = /<script(?:\s[^>]*)?>([\s\S]*?)<\/script>/gi;
+    const inner = new RegExp(source.inline_script_regex);
+    let m: RegExpExecArray | null;
+    scriptPattern.lastIndex = 0;
+    while ((m = scriptPattern.exec(html)) !== null) {
+      const hit = m[1].match(inner);
+      if (hit && hit[1]) return hit[1];
+    }
+    return undefined;
+  }
+
+  return undefined;
+}
+
+export function extractTokenFromBundle(
+  source: AuthTokenSource,
+  bundleContent: string,
+): string | undefined {
+  if (source.kind !== "js-bundle" || !source.bundle_regex) return undefined;
+  const m = bundleContent.match(new RegExp(source.bundle_regex));
+  return m?.[1];
+}
+
+// ─── Endpoint enrichment ──────────────────────────────────────────────────
+//
+// Walks captured requests, extracts auth-sensitive headers, runs findTokenSources
+// for each, and attaches matching TokenBindings to the corresponding endpoints.
+//
+// This is the capture-side glue that wires HTML/JS token discovery into the DAG.
+
+import type { EndpointDescriptor, AuthTokenBinding, AuthTokenSource as _TS } from "../types/index.js";
+
+/** Header names that commonly carry site-computed tokens worth scanning for.
+ *  Bearer auth, CSRF tokens, and transaction IDs are the prime targets. */
+const TOKEN_HEADER_PATTERN =
+  /^(authorization|x-csrf-token|x-xsrf-token|x-csrftoken|csrf-token|x-api-key|x-auth-token|x-access-token|x-session-token|x-shopify-storefront-access-token|x-twitter-auth-type|x-client-transaction-id|x-requested-with|x-client-version)$/i;
+
+interface MinimalRequest {
+  url: string;
+  method: string;
+  request_headers: Record<string, string>;
+}
+
+/**
+ * For each captured request that uses a token-bearing header, find that token's
+ * source locations in the captured HTML / JS bundles and attach AuthTokenBinding
+ * entries to the endpoint that matches the request URL + method.
+ *
+ * Runs in-place: mutates endpoint.auth_tokens.
+ */
+export function enrichEndpointsWithTokenSources(
+  endpoints: EndpointDescriptor[],
+  requests: MinimalRequest[],
+  html: string | undefined,
+  jsBundles: Map<string, string> | undefined,
+): number {
+  if (!html && (!jsBundles || jsBundles.size === 0)) return 0;
+
+  let enriched = 0;
+  for (const req of requests) {
+    const matching = endpoints.filter((ep) => endpointMatchesRequest(ep, req));
+    if (matching.length === 0) continue;
+
+    for (const [headerName, headerValue] of Object.entries(req.request_headers)) {
+      if (!TOKEN_HEADER_PATTERN.test(headerName)) continue;
+      // Bearer tokens are prefixed — strip before scanning
+      const tokenValue = extractTokenValue(headerName, headerValue);
+      if (!tokenValue) continue;
+
+      const sources = findTokenSources(tokenValue, html, jsBundles);
+      if (sources.length === 0) continue;
+
+      const binding: AuthTokenBinding = {
+        param_name: headerName.toLowerCase(),
+        param_location: "header",
+        sources,
+        refresh_on_401: true,
+      };
+
+      for (const ep of matching) {
+        if (!ep.auth_tokens) ep.auth_tokens = [];
+        // Dedupe: skip if a binding for the same param already exists
+        if (ep.auth_tokens.some((b) => b.param_name === binding.param_name)) continue;
+        ep.auth_tokens.push(binding);
+        enriched++;
+      }
+    }
+  }
+  return enriched;
+}
+
+/** For Authorization: Bearer XYZ → returns "XYZ". For plain headers → returns value as-is. */
+function extractTokenValue(headerName: string, headerValue: string): string | null {
+  if (!headerValue) return null;
+  if (headerName.toLowerCase() === "authorization") {
+    const parts = headerValue.split(/\s+/, 2);
+    if (parts.length === 2 && /^(bearer|token)$/i.test(parts[0])) return parts[1];
+    return headerValue;
+  }
+  return headerValue;
+}
+
+/** Loose match: compare hostname + path prefix of the captured request against the endpoint's url_template. */
+function endpointMatchesRequest(ep: EndpointDescriptor, req: MinimalRequest): boolean {
+  if (ep.method !== req.method) return false;
+  try {
+    const reqUrl = new URL(req.url);
+    const tplUrl = new URL(ep.url_template.replace(/\{[^}]+\}/g, "x"));
+    if (reqUrl.hostname !== tplUrl.hostname) return false;
+    // Compare path segment counts + non-template segments
+    const reqSegs = reqUrl.pathname.split("/").filter(Boolean);
+    const tplSegs = tplUrl.pathname.split("/").filter(Boolean);
+    if (reqSegs.length !== tplSegs.length) return false;
+    for (let i = 0; i < tplSegs.length; i++) {
+      if (tplSegs[i] === "x") continue; // was a {template} placeholder
+      if (tplSegs[i] !== reqSegs[i]) return false;
+    }
+    return true;
+  } catch {
+    return false;
+  }
+}

package/runtime-src/types/skill.ts

@@ -130,6 +130,23 @@ export interface EndpointPathBindingCandidate {
   matched_page_hint?: boolean;
 }
 
+export interface AuthTokenSource {
+  kind: "cookie" | "html-meta" | "html-inline-script" | "js-bundle" | "observed-request";
+  cookie_names?: string[];
+  meta_name?: string;
+  meta_attr?: "content" | "value";
+  inline_script_regex?: string;
+  bundle_url_pattern?: string;
+  bundle_regex?: string;
+}
+
+export interface AuthTokenBinding {
+  param_name: string;
+  param_location: "header" | "body" | "query";
+  sources: AuthTokenSource[];
+  refresh_on_401?: boolean;
+}
+
 export interface EndpointDescriptor {
   endpoint_id: string;
   method: "GET" | "POST" | "PUT" | "PATCH" | "DELETE" | "HEAD" | "OPTIONS" | "WS";
@@ -146,6 +163,8 @@ export interface EndpointDescriptor {
   body?: Record<string, unknown>;
   csrf_plan?: CsrfPlan;
   oauth_plan?: OAuthPlan;
+  /** Token bindings discovered during capture — maps auth headers to their page sources */
+  auth_tokens?: AuthTokenBinding[];
   transform_ref?: string;
   idempotency: Idempotency;
   verification_status: VerificationStatus;

package/vendor/kuri/manifest.json

@@ -1,24 +1,28 @@
 {
   "repo_url": "https://github.com/justrach/kuri.git",
   "branch": "adding-extensions",
-  "source_sha": "005ce5cdf41347b683df99d307868a11439be972",
-  "built_at": "2026-04-04T11:57:45.378Z",
+  "source_sha": "08eecbe3740f046a46f656eed7ebfc66c1bad9bb",
+  "built_at": "2026-04-05T06:43:57.212Z",
   "binaries": {
     "darwin-arm64": {
       "zig_target": "aarch64-macos",
-      "sha256": "44d5e985e075ba57c7cc8e3ab51a39186b4bf62ece4d0c2e57797cbd4d0408d2"
+      "sha256": "1553633e722d18059dedffa8a52d55ed6c052e4961fd2753ee0b62be60b241bf"
     },
     "darwin-x64": {
       "zig_target": "x86_64-macos",
-      "sha256": "09dc2104449506f2ba8a73edc884bb7399c349c12fc6ec6b3d5f7f652918742f"
+      "sha256": "b5eb07e631c6ddad64019c8d0c86c32cb76a74ff0791ac5611a3aa3550767ec8"
     },
     "linux-arm64": {
       "zig_target": "aarch64-linux",
-      "sha256": "98c18cec5a588f44746cd465b0c8fd17cec311ca03e1292dab173841db8bfcf1"
+      "sha256": "ea88a26f7b335d5842b0c1d83bfa4066bed0a119284560f6bd3833f1d240cce2"
     },
     "linux-x64": {
       "zig_target": "x86_64-linux",
-      "sha256": "6d081f758ee76e44bbf1bb9c633057212f9c1641d97060e391dadf35015fbbe2"
+      "sha256": "175a7c59e458e952a26974f0fb5c2ce374e56f2c4c352903b481b5aa5a16978f"
+    },
+    "win-x64": {
+      "zig_target": "x86_64-windows",
+      "sha256": "176291ad9827a183ba7322ddb56cc1fa5edc7c214a264ecdf8a1d5d18366d686"
     }
   }
 }