unbrowse 2.9.0 → 2.9.1

package/dist/cli.js

@@ -501,11 +501,75 @@ async function getCookies(tabId) {
   const raw = await kuriGet("/cookies", { tab_id: tabId });
   return raw?.result?.cookies ?? [];
 }
+async function setCookieViaCDP(wsUrl, cookie) {
+  return new Promise((resolve) => {
+    const timer = setTimeout(() => {
+      resolve(false);
+    }, 3000);
+    try {
+      const ws = new (__require("ws"))(wsUrl);
+      ws.on("open", () => {
+        ws.send(JSON.stringify({
+          id: 1,
+          method: "Network.setCookie",
+          params: {
+            ...cookie,
+            url: `https://${cookie.domain.replace(/^\./, "")}/`
+          }
+        }));
+      });
+      ws.on("message", (data) => {
+        clearTimeout(timer);
+        try {
+          const msg = JSON.parse(data.toString());
+          if (msg.id === 1) {
+            ws.close();
+            resolve(msg.result?.success ?? false);
+          }
+        } catch {
+          ws.close();
+          resolve(false);
+        }
+      });
+      ws.on("error", () => {
+        clearTimeout(timer);
+        resolve(false);
+      });
+    } catch {
+      clearTimeout(timer);
+      resolve(false);
+    }
+  });
+}
 async function setCookie(tabId, cookie) {
+  const value = cookie.value.replace(/^"|"$/g, "");
+  if (cookie.secure || cookie.httpOnly) {
+    try {
+      const res = await fetch("http://127.0.0.1:9222/json", { signal: AbortSignal.timeout(1000) }).catch(() => null);
+      if (res?.ok) {
+        const pages = await res.json();
+        const page = pages.find((p) => p.id === tabId);
+        if (page?.webSocketDebuggerUrl) {
+          const success = await setCookieViaCDP(page.webSocketDebuggerUrl, {
+            name: cookie.name,
+            value,
+            domain: cookie.domain,
+            path: cookie.path || "/",
+            secure: cookie.secure ?? false,
+            httpOnly: cookie.httpOnly ?? false,
+            sameSite: cookie.sameSite || "Lax",
+            ...cookie.expires && cookie.expires > 0 ? { expires: cookie.expires } : {}
+          });
+          if (success)
+            return;
+        }
+      }
+    } catch {}
+  }
   await kuriGet("/cookies", {
     tab_id: tabId,
     name: cookie.name,
-    value: cookie.value,
+    value,
     domain: cookie.domain,
     ...cookie.path ? { path: cookie.path } : {}
   });
@@ -4596,6 +4660,7 @@ __export(exports_client2, {
   publishGraphEdges: () => publishGraphEdges,
   normalizeAgentEmail: () => normalizeAgentEmail2,
   listSkills: () => listSkills,
+  isX402Error: () => isX402Error,
   isValidAgentEmail: () => isValidAgentEmail2,
   isLocalOnlyMode: () => isLocalOnlyMode,
   hashApiKey: () => hashApiKey,
@@ -4638,6 +4703,9 @@ function decodeBase64Json2(value) {
     return;
   }
 }
+function isX402Error(err) {
+  return !!err && typeof err === "object" && err.x402 === true;
+}
 function scopedSkillKey(skillId, scopeId) {
   return scopeId ? `${scopeId}:${skillId}` : skillId;
 }
@@ -5142,10 +5210,20 @@ async function searchIntentResolve(intent, domain, domainK = 5, globalK = 10) {
       domain_k: domainK,
       global_k: globalK
     });
-  } catch {
+  } catch (err) {
+    if (isX402Error(err))
+      throw err;
     const [domain_results, global_results] = await Promise.all([
-      domain ? searchIntentInDomain(intent, domain, domainK).catch(() => []) : Promise.resolve([]),
-      searchIntent(intent, globalK).catch(() => [])
+      domain ? searchIntentInDomain(intent, domain, domainK).catch((fallbackErr) => {
+        if (isX402Error(fallbackErr))
+          throw fallbackErr;
+        return [];
+      }) : Promise.resolve([]),
+      searchIntent(intent, globalK).catch((fallbackErr) => {
+        if (isX402Error(fallbackErr))
+          throw fallbackErr;
+        return [];
+      })
     ]);
     return { domain_results, global_results, skipped_global: false };
   }
@@ -15053,17 +15131,51 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
   const MARKETPLACE_TIMEOUT_MS = context?.url ? 5000 : 30000;
   if (!forceCapture) {
     const ts0 = Date.now();
-    const { domain_results: domainResults, global_results: globalResults } = await Promise.race([
-      searchIntentResolve(queryIntent, requestedDomain ?? undefined, MARKETPLACE_DOMAIN_SEARCH_K, MARKETPLACE_GLOBAL_SEARCH_K),
-      new Promise((resolve) => setTimeout(() => {
-        console.log(`[marketplace] timeout after ${MARKETPLACE_TIMEOUT_MS}ms — falling through to browser`);
-        resolve({ domain_results: [], global_results: [], skipped_global: true });
-      }, MARKETPLACE_TIMEOUT_MS))
-    ]).catch(() => ({
-      domain_results: [],
-      global_results: [],
-      skipped_global: false
-    }));
+    let searchResponse;
+    try {
+      searchResponse = await Promise.race([
+        searchIntentResolve(queryIntent, requestedDomain ?? undefined, MARKETPLACE_DOMAIN_SEARCH_K, MARKETPLACE_GLOBAL_SEARCH_K),
+        new Promise((resolve) => setTimeout(() => {
+          console.log(`[marketplace] timeout after ${MARKETPLACE_TIMEOUT_MS}ms — falling through to browser`);
+          resolve({ domain_results: [], global_results: [], skipped_global: true });
+        }, MARKETPLACE_TIMEOUT_MS))
+      ]);
+    } catch (err) {
+      if (isX402Error(err)) {
+        const trace2 = {
+          trace_id: nanoid7(),
+          skill_id: "marketplace-search",
+          endpoint_id: "search",
+          started_at: new Date().toISOString(),
+          completed_at: new Date().toISOString(),
+          success: false,
+          status_code: 402,
+          error: "payment_required"
+        };
+        return {
+          result: {
+            error: "payment_required",
+            payment_status: "payment_required",
+            wallet_provider: "lobster.cash",
+            message: "Marketplace search requires payment before shared graph results are returned.",
+            next_step: "Pay the Tier 3 search fee, or re-run with force capture for free local discovery.",
+            indexing_fallback_available: true,
+            tier: "tier3",
+            terms: err.terms
+          },
+          trace: trace2,
+          source: "marketplace",
+          skill: undefined,
+          timing: finalize("marketplace", null, undefined, undefined, trace2)
+        };
+      }
+      searchResponse = {
+        domain_results: [],
+        global_results: [],
+        skipped_global: false
+      };
+    }
+    const { domain_results: domainResults, global_results: globalResults } = searchResponse;
     timing.search_ms = Date.now() - ts0;
     console.log(`[marketplace] search: ${domainResults.length} domain + ${globalResults.length} global results (${timing.search_ms}ms)`);
     const seen = new Set;
@@ -16995,6 +17107,7 @@ async function registerRoutes(app) {
     if (session.domain && session.domain !== newDomain) {
       await authProfileSave(session.tabId, session.domain).catch(() => {});
     }
+    let cookiesInjected = 0;
     if (newDomain && newDomain !== session.domain) {
       await authProfileLoad(session.tabId, newDomain).catch(() => {});
       try {
@@ -17003,6 +17116,7 @@ async function registerRoutes(app) {
           for (const c of browserCookies) {
             await setCookie(session.tabId, c).catch(() => {});
           }
+          cookiesInjected = browserCookies.length;
         }
       } catch {}
     }
@@ -17015,7 +17129,7 @@ async function registerRoutes(app) {
     session.url = typeof finalUrl === "string" && finalUrl.startsWith("http") ? finalUrl : url;
     session.domain = profileName(session.url);
     await injectInterceptor(session.tabId);
-    return reply.send({ ok: true, url: session.url, tab_id: session.tabId, auth_profile: session.domain });
+    return reply.send({ ok: true, url: session.url, tab_id: session.tabId, auth_profile: session.domain, ...cookiesInjected > 0 ? { cookies_injected: cookiesInjected } : {} });
   });
   app.post("/v1/browse/snap", async (req, reply) => {
     const { filter } = req.body ?? {};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "2.9.0",
+  "version": "2.9.1",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {

package/runtime-src/api/routes.ts

@@ -845,6 +845,7 @@ export async function registerRoutes(app: FastifyInstance) {
     }
 
     // Inject cookies: try Kuri auth profile first, fall back to Chrome SQLite extraction
+    let cookiesInjected = 0;
     if (newDomain && newDomain !== session.domain) {
       await kuri.authProfileLoad(session.tabId, newDomain).catch(() => {});
 
@@ -855,6 +856,7 @@ export async function registerRoutes(app: FastifyInstance) {
           for (const c of browserCookies) {
             await kuri.setCookie(session.tabId, c).catch(() => {});
           }
+          cookiesInjected = browserCookies.length;
         }
       } catch { /* non-fatal */ }
     }
@@ -872,7 +874,7 @@ export async function registerRoutes(app: FastifyInstance) {
     // Re-inject interceptor via evaluate for current page context
     await injectInterceptor(session.tabId); // chunked injection for current page
 
-    return reply.send({ ok: true, url: session.url, tab_id: session.tabId, auth_profile: session.domain });
+    return reply.send({ ok: true, url: session.url, tab_id: session.tabId, auth_profile: session.domain, ...(cookiesInjected > 0 ? { cookies_injected: cookiesInjected } : {}) });
   });
 
   // POST /v1/browse/snap — a11y snapshot

package/runtime-src/client/index.ts

@@ -28,6 +28,10 @@ function decodeBase64Json(value: string): unknown {
   }
 }
 
+export function isX402Error(err: unknown): err is Error & { x402: true; terms?: unknown; status?: number } {
+  return !!err && typeof err === "object" && (err as { x402?: unknown }).x402 === true;
+}
+
 function scopedSkillKey(skillId: string, scopeId?: string): string {
   return scopeId ? `${scopeId}:${skillId}` : skillId;
 }
@@ -678,12 +682,19 @@ export async function searchIntentResolve(
       domain_k: domainK,
       global_k: globalK,
     });
-  } catch {
+  } catch (err) {
+    if (isX402Error(err)) throw err;
     const [domain_results, global_results] = await Promise.all([
       domain
-        ? searchIntentInDomain(intent, domain, domainK).catch(() => [] as Array<{ id: number; score: number; metadata: Record<string, unknown> }>)
+        ? searchIntentInDomain(intent, domain, domainK).catch((fallbackErr) => {
+            if (isX402Error(fallbackErr)) throw fallbackErr;
+            return [] as Array<{ id: number; score: number; metadata: Record<string, unknown> }>;
+          })
         : Promise.resolve([] as Array<{ id: number; score: number; metadata: Record<string, unknown> }>),
-      searchIntent(intent, globalK).catch(() => [] as Array<{ id: number; score: number; metadata: Record<string, unknown> }>),
+      searchIntent(intent, globalK).catch((fallbackErr) => {
+        if (isX402Error(fallbackErr)) throw fallbackErr;
+        return [] as Array<{ id: number; score: number; metadata: Record<string, unknown> }>;
+      }),
     ]);
     return { domain_results, global_results, skipped_global: false };
   }

package/runtime-src/kuri/client.ts

@@ -526,12 +526,78 @@ export async function getCookies(tabId: string): Promise<KuriCookie[]> {
   return raw?.result?.cookies ?? [];
 }
 
-/** Set a single cookie. */
+/** Set a cookie via raw CDP WebSocket — supports all cookie attributes (secure, httpOnly, sameSite, expires). */
+async function setCookieViaCDP(wsUrl: string, cookie: {
+  name: string; value: string; domain: string; path: string;
+  secure: boolean; httpOnly: boolean; sameSite: string; expires?: number;
+}): Promise<boolean> {
+  return new Promise((resolve) => {
+    const timer = setTimeout(() => { resolve(false); }, 3000);
+    try {
+      const ws = new (require("ws") as typeof import("ws"))(wsUrl);
+      ws.on("open", () => {
+        ws.send(JSON.stringify({
+          id: 1,
+          method: "Network.setCookie",
+          params: {
+            ...cookie,
+            url: `https://${cookie.domain.replace(/^\./, "")}/`,
+          },
+        }));
+      });
+      ws.on("message", (data: Buffer) => {
+        clearTimeout(timer);
+        try {
+          const msg = JSON.parse(data.toString());
+          if (msg.id === 1) {
+            ws.close();
+            resolve(msg.result?.success ?? false);
+          }
+        } catch { ws.close(); resolve(false); }
+      });
+      ws.on("error", () => { clearTimeout(timer); resolve(false); });
+    } catch { clearTimeout(timer); resolve(false); }
+  });
+}
+
+/** Set a single cookie via raw CDP (Chrome debug port) for full attribute support.
+ *  Falls back to Kuri's /cookies endpoint if CDP is unavailable. */
+/** Set a single cookie via raw CDP (Chrome debug port) for full attribute support.
+ *  Falls back to Kuri's /cookies endpoint if CDP is unavailable. */
 export async function setCookie(tabId: string, cookie: KuriCookie): Promise<void> {
+  // Strip wrapping quotes from cookie values (Chrome stores some values like JSESSIONID with literal quotes)
+  const value = cookie.value.replace(/^"|"$/g, "");
+
+  // Try raw CDP first — Kuri's /cookies endpoint doesn't pass secure/httpOnly/sameSite/expires
+  // which causes auth failures on sites like LinkedIn that require secure cookies.
+  if (cookie.secure || cookie.httpOnly) {
+    try {
+      const res = await fetch("http://127.0.0.1:9222/json", { signal: AbortSignal.timeout(1000) }).catch(() => null);
+      if (res?.ok) {
+        const pages = await res.json() as Array<{ id: string; webSocketDebuggerUrl?: string }>;
+        const page = pages.find(p => p.id === tabId);
+        if (page?.webSocketDebuggerUrl) {
+          const success = await setCookieViaCDP(page.webSocketDebuggerUrl, {
+            name: cookie.name,
+            value,
+            domain: cookie.domain,
+            path: cookie.path || "/",
+            secure: cookie.secure ?? false,
+            httpOnly: cookie.httpOnly ?? false,
+            sameSite: cookie.sameSite || "Lax",
+            ...(cookie.expires && cookie.expires > 0 ? { expires: cookie.expires } : {}),
+          });
+          if (success) return;
+        }
+      }
+    } catch { /* CDP unavailable, fall through to Kuri */ }
+  }
+
+  // Fallback: Kuri's /cookies endpoint (no secure/httpOnly support)
   await kuriGet("/cookies", {
     tab_id: tabId,
     name: cookie.name,
-    value: cookie.value,
+    value,
     domain: cookie.domain,
     ...(cookie.path ? { path: cookie.path } : {}),
   });

package/runtime-src/orchestrator/index.ts

@@ -1,4 +1,4 @@
-import { searchIntentResolve, recordOrchestrationPerf } from "../client/index.js";
+import { isX402Error, searchIntentResolve, recordOrchestrationPerf } from "../client/index.js";
 import * as kuri from "../kuri/client.js";
 import { emitRouteTrace, recordFailure } from "../telemetry.js";
 import { publishSkill, getSkill } from "../marketplace/index.js";
@@ -2931,24 +2931,62 @@ export async function resolveAndExecute(
     // 1. Search marketplace — single remote call, capped by timeout when URL available
     const ts0 = Date.now();
     type SearchResult = { id: number; score: number; metadata: Record<string, unknown> };
-    const { domain_results: domainResults, global_results: globalResults } = await Promise.race([
-      searchIntentResolve(
-        queryIntent,
-        requestedDomain ?? undefined,
-        MARKETPLACE_DOMAIN_SEARCH_K,
-        MARKETPLACE_GLOBAL_SEARCH_K,
-      ),
-      new Promise<{ domain_results: SearchResult[]; global_results: SearchResult[]; skipped_global: boolean }>((resolve) =>
-        setTimeout(() => {
-          console.log(`[marketplace] timeout after ${MARKETPLACE_TIMEOUT_MS}ms — falling through to browser`);
-          resolve({ domain_results: [], global_results: [], skipped_global: true });
-        }, MARKETPLACE_TIMEOUT_MS),
-      ),
-    ]).catch(() => ({
-      domain_results: [] as SearchResult[],
-      global_results: [] as SearchResult[],
-      skipped_global: false,
-    }));
+    let searchResponse: {
+      domain_results: SearchResult[];
+      global_results: SearchResult[];
+      skipped_global: boolean;
+    };
+    try {
+      searchResponse = await Promise.race([
+        searchIntentResolve(
+          queryIntent,
+          requestedDomain ?? undefined,
+          MARKETPLACE_DOMAIN_SEARCH_K,
+          MARKETPLACE_GLOBAL_SEARCH_K,
+        ),
+        new Promise<{ domain_results: SearchResult[]; global_results: SearchResult[]; skipped_global: boolean }>((resolve) =>
+          setTimeout(() => {
+            console.log(`[marketplace] timeout after ${MARKETPLACE_TIMEOUT_MS}ms — falling through to browser`);
+            resolve({ domain_results: [], global_results: [], skipped_global: true });
+          }, MARKETPLACE_TIMEOUT_MS),
+        ),
+      ]);
+    } catch (err) {
+      if (isX402Error(err)) {
+        const trace: ExecutionTrace = {
+          trace_id: nanoid(),
+          skill_id: "marketplace-search",
+          endpoint_id: "search",
+          started_at: new Date().toISOString(),
+          completed_at: new Date().toISOString(),
+          success: false,
+          status_code: 402,
+          error: "payment_required",
+        };
+        return {
+          result: {
+            error: "payment_required",
+            payment_status: "payment_required",
+            wallet_provider: "lobster.cash",
+            message: "Marketplace search requires payment before shared graph results are returned.",
+            next_step: "Pay the Tier 3 search fee, or re-run with force capture for free local discovery.",
+            indexing_fallback_available: true,
+            tier: "tier3",
+            terms: err.terms,
+          },
+          trace,
+          source: "marketplace",
+          skill: undefined as any,
+          timing: finalize("marketplace", null, undefined, undefined, trace),
+        };
+      }
+      searchResponse = {
+        domain_results: [] as SearchResult[],
+        global_results: [] as SearchResult[],
+        skipped_global: false,
+      };
+    }
+    const { domain_results: domainResults, global_results: globalResults } = searchResponse;
     timing.search_ms = Date.now() - ts0;
     console.log(`[marketplace] search: ${domainResults.length} domain + ${globalResults.length} global results (${timing.search_ms}ms)`);