unbrowse 3.6.0 → 3.7.0-preview.1

package/dist/cli.js

@@ -31,7 +31,7 @@ var __promiseAll = (args) => Promise.all(args);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.6.0", BUILD_GIT_SHA = "d598e34517f7", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy42LjAiLCJnaXRfc2hhIjoiZDU5OGUzNDUxN2Y3IiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUBkNTk4ZTM0NTE3ZjciLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA5VDIyOjU0OjE2LjkyNVoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "QYlGIFDhKbCtaXoeeGCG8eJde-VnRFlXyNOJanZJ1Mk", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.7.0-preview.1", BUILD_GIT_SHA = "b3ecde042c51", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6ImIzZWNkZTA0MmM1MSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYjNlY2RlMDQyYzUxIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNToxNDowNy4xMDRaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "BQOPfluuXHVkQx0LryOJrfGDa-ctFVdDBcNhh53WHQQ", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash } from "crypto";

package/dist/mcp.js

@@ -225,11 +225,11 @@ import { dirname, join, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.6.0";
-var BUILD_GIT_SHA = "d598e34517f7";
+var BUILD_RELEASE_VERSION = "3.7.0-preview.1";
+var BUILD_GIT_SHA = "b3ecde042c51";
 var BUILD_CODE_HASH = "5d9ebf619c61";
-var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy42LjAiLCJnaXRfc2hhIjoiZDU5OGUzNDUxN2Y3IiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUBkNTk4ZTM0NTE3ZjciLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA5VDIyOjU0OjE2LjkyNVoifQ";
-var BUILD_RELEASE_MANIFEST_SIGNATURE = "QYlGIFDhKbCtaXoeeGCG8eJde-VnRFlXyNOJanZJ1Mk";
+var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6ImIzZWNkZTA0MmM1MSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYjNlY2RlMDQyYzUxIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNToxNDowNy4xMDRaIn0";
+var BUILD_RELEASE_MANIFEST_SIGNATURE = "BQOPfluuXHVkQx0LryOJrfGDa-ctFVdDBcNhh53WHQQ";
 var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts

package/dist/server.js

@@ -3827,6 +3827,13 @@ __export(exports_reverse_engineer, {
   extractAuthHeaders: () => extractAuthHeaders
 });
 import { nanoid as nanoid2 } from "nanoid";
+import { createHash } from "node:crypto";
+function stableEndpointId(method, urlTemplate) {
+  if (!method || !urlTemplate)
+    return nanoid2();
+  const hash = createHash("sha256").update(`${method}:${urlTemplate}`).digest("base64url");
+  return hash.slice(0, 21);
+}
 function extractGraphQLOperationName(url, requestBody) {
   const urlMatch = url.match(/\/graphql\/[^/]+\/(\w+)/);
   if (urlMatch)
@@ -4447,10 +4454,11 @@ function extractEndpoints(requests, wsMessages, context) {
     });
     const csrfPlan = inferCsrfPlan(req, parsedRequestBody);
     const endpointGraphqlOp = /graphql/i.test(req.url) ? extractGraphQLOperationName(req.url, req.request_body) : undefined;
+    const computedUrlTemplate = qTemplateStr ? `${pathTemplate}${pathTemplate.includes("?") ? "&" : "?"}${qTemplateStr}` : pathTemplate;
     let endpoint = {
-      endpoint_id: nanoid2(),
+      endpoint_id: stableEndpointId(req.method, computedUrlTemplate),
       method: req.method,
-      url_template: qTemplateStr ? `${pathTemplate}${pathTemplate.includes("?") ? "&" : "?"}${qTemplateStr}` : pathTemplate,
+      url_template: computedUrlTemplate,
       description: buildEndpointDescription(req, sampleRequest, sampleResponse),
       headers_template: sanitizeHeaders(req.request_headers),
       query: sanitizedQParams,
@@ -4541,7 +4549,7 @@ function extractEndpoints(requests, wsMessages, context) {
         response_schema = inferSchema(jsonSamples);
       }
       const endpoint = {
-        endpoint_id: nanoid2(),
+        endpoint_id: stableEndpointId("WS", wsUrl),
         method: "WS",
         url_template: wsUrl,
         idempotency: "safe",
@@ -7096,10 +7104,10 @@ var init_capture = __esm(async () => {
 });
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.6.0", BUILD_GIT_SHA = "d598e34517f7", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy42LjAiLCJnaXRfc2hhIjoiZDU5OGUzNDUxN2Y3IiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUBkNTk4ZTM0NTE3ZjciLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA5VDIyOjU0OjE2LjkyNVoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "QYlGIFDhKbCtaXoeeGCG8eJde-VnRFlXyNOJanZJ1Mk", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.7.0-preview.1", BUILD_GIT_SHA = "b3ecde042c51", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6ImIzZWNkZTA0MmM1MSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYjNlY2RlMDQyYzUxIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNToxNDowNy4xMDRaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "BQOPfluuXHVkQx0LryOJrfGDa-ctFVdDBcNhh53WHQQ", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
-import { createHash } from "crypto";
+import { createHash as createHash2 } from "crypto";
 import { existsSync as existsSync4, readFileSync as readFileSync2, readdirSync } from "fs";
 import { dirname, join as join3, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
@@ -7116,7 +7124,7 @@ function collectTsFiles(dir) {
   return results;
 }
 function hashFiles(srcDir, files) {
-  const hash = createHash("sha256");
+  const hash = createHash2("sha256");
   for (const file of files) {
     hash.update(file.slice(srcDir.length));
     hash.update(readFileSync2(file, "utf-8"));
@@ -7156,7 +7164,7 @@ function computeCodeHash() {
   } catch {}
   const pkgVersion = getPackageVersion();
   if (pkgVersion !== "unknown") {
-    return createHash("sha256").update(`package:${pkgVersion}`).digest("hex").slice(0, 12);
+    return createHash2("sha256").update(`package:${pkgVersion}`).digest("hex").slice(0, 12);
   }
   return "compiled";
 }
@@ -7210,13 +7218,13 @@ var init_version = __esm(() => {
 });
 
 // ../../src/payments/cascade.ts
-import { createHash as createHash2 } from "crypto";
+import { createHash as createHash3 } from "crypto";
 import bs58 from "bs58";
 function payableContributors(skill) {
   return (skill.contributors ?? []).filter((c) => !!c.wallet_address?.trim());
 }
 function cascadeLabel(skillId) {
-  const digest = createHash2("sha256").update(skillId).digest("hex");
+  const digest = createHash3("sha256").update(skillId).digest("hex");
   return `ubr-${digest.slice(0, 23)}`;
 }
 function decodeSecretKey(raw) {
@@ -7610,7 +7618,7 @@ __export(exports_client2, {
 import { readFileSync as readFileSync4, writeFileSync as writeFileSync3, existsSync as existsSync7, mkdirSync as mkdirSync4, readdirSync as readdirSync2, unlinkSync } from "fs";
 import { join as join6 } from "path";
 import { homedir as homedir4, hostname, release as osRelease } from "os";
-import { randomBytes as randomBytes2, createHash as createHash3 } from "crypto";
+import { randomBytes as randomBytes2, createHash as createHash4 } from "crypto";
 import { createInterface } from "readline";
 import { execSync } from "child_process";
 function buildReleaseAttestationHeaders(manifestBase64, signature) {
@@ -7887,7 +7895,7 @@ function getApiKey() {
 function hashApiKey(key) {
   if (!key || key === "local-only")
     return "";
-  return createHash3("sha256").update(key).digest("hex");
+  return createHash4("sha256").update(key).digest("hex");
 }
 function getAgentId() {
   const config = loadConfig();
@@ -9069,7 +9077,7 @@ var init_publish_admission = __esm(() => {
 });
 
 // ../../src/telemetry.ts
-import { createHash as createHash4 } from "node:crypto";
+import { createHash as createHash5 } from "node:crypto";
 import { existsSync as existsSync8, mkdirSync as mkdirSync5, writeFileSync as writeFileSync4 } from "node:fs";
 import { join as join7 } from "node:path";
 function getTraceDir() {
@@ -9079,7 +9087,7 @@ function isTracingEnabled() {
   return process.env.UNBROWSE_DISABLE_TRACES !== "1";
 }
 function hashValue(value) {
-  return createHash4("sha256").update(value).digest("hex").slice(0, 16);
+  return createHash5("sha256").update(value).digest("hex").slice(0, 16);
 }
 function isSensitiveName(name) {
   return SENSITIVE_PATTERNS.some((re) => re.test(name));
@@ -9097,7 +9105,7 @@ function anonymizeUrl(url) {
 }
 function hashResponseBody(result) {
   const str = typeof result === "string" ? result : JSON.stringify(result ?? "");
-  return createHash4("sha256").update(str).digest("hex").slice(0, 32);
+  return createHash5("sha256").update(str).digest("hex").slice(0, 32);
 }
 function classifyFailure(error) {
   if (!error)
@@ -15495,7 +15503,12 @@ __export(exports_execution, {
   buildCanonicalDocumentEndpoint: () => buildCanonicalDocumentEndpoint
 });
 import { nanoid as nanoid6 } from "nanoid";
-import { createHash as createHash5 } from "node:crypto";
+import { createHash as createHash6 } from "node:crypto";
+function stableEndpointId2(method, urlTemplate) {
+  if (!method || !urlTemplate)
+    return nanoid6();
+  return createHash6("sha256").update(`${method}:${urlTemplate}`).digest("base64url").slice(0, 21);
+}
 function stampTrace(trace) {
   trace.trace_version = TRACE_VERSION;
   return trace;
@@ -16039,10 +16052,11 @@ function buildPageArtifactCapture(url, intent, html, authRequired = false) {
   const searchForms = detectSearchForms(html);
   const validSearchForm = searchForms.find((spec) => isStructuredSearchForm(spec));
   const response_schema = inferSchema([extracted.data]);
+  const computedTemplate = templatizeQueryParams(url);
   const endpoint = {
-    endpoint_id: nanoid6(),
+    endpoint_id: stableEndpointId2("GET", computedTemplate),
     method: "GET",
-    url_template: templatizeQueryParams(url),
+    url_template: computedTemplate,
     idempotency: "safe",
     verification_status: "verified",
     reliability_score: extracted.confidence,
@@ -16090,7 +16104,7 @@ function buildCanonicalDocumentEndpoint(url, intent, authRequired = false) {
   const replayTemplate = deriveStructuredDataReplayTemplate(url);
   if (replayUrl === url && replayTemplate === url)
     return;
-  const canonicalId = createHash5("sha1").update(replayTemplate !== url ? replayTemplate : replayUrl).digest("base64url").slice(0, 21);
+  const canonicalId = createHash6("sha1").update(replayTemplate !== url ? replayTemplate : replayUrl).digest("base64url").slice(0, 21);
   const endpoint = {
     endpoint_id: canonicalId,
     method: "GET",
@@ -16595,7 +16609,7 @@ async function executeBrowserCapture(skill, params, options) {
         epUrl = `${route.url}?${qStr}`;
       }
       endpoints.push({
-        endpoint_id: nanoid6(),
+        endpoint_id: stableEndpointId2("GET", epUrl),
         method: "GET",
         url_template: epUrl,
         query: epQuery,
@@ -19117,10 +19131,10 @@ var init_timing_economics = __esm(() => {
 });
 
 // ../../src/routing-telemetry.ts
-import { createHash as createHash6 } from "node:crypto";
+import { createHash as createHash7 } from "node:crypto";
 import { nanoid as nanoid8 } from "nanoid";
 function stableHash(value) {
-  return createHash6("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 24);
+  return createHash7("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 24);
 }
 function sanitizeScalar(value) {
   if (value == null)
@@ -19474,7 +19488,7 @@ function applyVerificationResults(skill, verification) {
 import { nanoid as nanoid9 } from "nanoid";
 import { existsSync as existsSync13, writeFileSync as writeFileSync8, readFileSync as readFileSync8, mkdirSync as mkdirSync9, readdirSync as readdirSync6 } from "node:fs";
 import { dirname as dirname2, join as join12 } from "node:path";
-import { createHash as createHash7 } from "node:crypto";
+import { createHash as createHash8 } from "node:crypto";
 function summarizeSchema(schema, maxDepth = 3) {
   function walk(s, depth) {
     if (depth <= 0)
@@ -19602,7 +19616,7 @@ function scopedResolveCacheKeys(scope, key) {
   return scope === "global" ? [scopedCacheKey("global", key)] : [scopedCacheKey(scope, key), scopedCacheKey("global", key)];
 }
 function snapshotPathForCacheKey(cacheKey) {
-  const digest = createHash7("sha1").update(cacheKey).digest("hex");
+  const digest = createHash8("sha1").update(cacheKey).digest("hex");
   return join12(SKILL_SNAPSHOT_DIR, `${digest}.json`);
 }
 function writeSkillSnapshot(cacheKey, skill) {
@@ -19928,7 +19942,12 @@ function withContextReplayEndpoint(skill, intent, contextUrl) {
     ...skill,
     endpoints: [canonical, ...skill.endpoints]
   };
-  cachePublishedSkill(augmented);
+  try {
+    const existing = findExistingSkillForDomain(skill.domain);
+    if (!existing || !existing.endpoints.some((ep) => ep.endpoint_id === canonical.endpoint_id)) {
+      cachePublishedSkill(augmented);
+    }
+  } catch {}
   return augmented;
 }
 function isSearchLikeIntent(intent, contextUrl) {
@@ -19948,7 +19967,7 @@ function buildLocalCanonicalReplaySkill(intent, contextUrl) {
   const domain = new URL(contextUrl).hostname.replace(/^www\./, "");
   const now = new Date().toISOString();
   const skill = {
-    skill_id: `canonical-${createHash7("sha1").update(contextUrl).digest("hex").slice(0, 12)}`,
+    skill_id: `canonical-${createHash8("sha1").update(contextUrl).digest("hex").slice(0, 12)}`,
     version: "1.0.0",
     schema_version: "1",
     name: `Canonical replay for ${domain}`,
@@ -22152,7 +22171,15 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
         continue;
       let endpointId = extractEndpointId(c.metadata) ?? undefined;
       if (endpointId && !skill.endpoints.some((ep) => ep.endpoint_id === endpointId)) {
-        endpointId = undefined;
+        const vecUrl = c.metadata?.url_template;
+        const urlMatch = vecUrl ? skill.endpoints.find((ep) => ep.url_template === vecUrl) : undefined;
+        if (urlMatch) {
+          console.log(`[marketplace] vecdb endpoint ${endpointId} stale → recovered via URL match: ${urlMatch.endpoint_id}`);
+          endpointId = urlMatch.endpoint_id;
+        } else {
+          console.log(`[marketplace] vecdb endpoint ${endpointId} not found on skill ${skillId} — dropping to skill-level match`);
+          endpointId = undefined;
+        }
       }
       ranked.push({
         candidate: c,
@@ -24194,7 +24221,13 @@ var init_browse_session = __esm(() => {
 
 // ../../src/api/browse-index.ts
 import { nanoid as nanoid10 } from "nanoid";
+import { createHash as createHash9 } from "node:crypto";
 import { readFileSync as readFileSync12 } from "node:fs";
+function stableEndpointId3(method, urlTemplate) {
+  if (!method || !urlTemplate)
+    return nanoid10();
+  return createHash9("sha256").update(`${method}:${urlTemplate}`).digest("base64url").slice(0, 21);
+}
 function normalizeBrowseUrl(url, baseUrl) {
   if (!url)
     return url;
@@ -24372,7 +24405,7 @@ async function cacheBrowseRequests(params) {
       return { domain, indexed: false, mode: "none", skill: null };
     const urlTemplate = templatizeQueryParams2(sessionUrl);
     const endpoint = {
-      endpoint_id: nanoid10(),
+      endpoint_id: stableEndpointId3("GET", urlTemplate),
       method: "GET",
       url_template: urlTemplate,
       idempotency: "safe",
@@ -26678,7 +26711,28 @@ async function registerRoutes(app) {
       ...context_url && typeof params?.url !== "string" ? { url: context_url } : {}
     };
     try {
-      const execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
+      let execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
+      if (!execResult.trace.success && execResult.result?.error === "endpoint_not_found" && typeof execParams.endpoint_id === "string") {
+        let recovered = false;
+        const freshSkill = await getSkill2(skill_id, clientScope);
+        if (freshSkill && freshSkill.endpoints.some((e) => e.endpoint_id === execParams.endpoint_id)) {
+          console.log(`[exec] endpoint ${execParams.endpoint_id} found in fresh marketplace skill — retrying`);
+          skill = freshSkill;
+          recovered = true;
+        }
+        if (!recovered && context_url) {
+          const { buildCanonicalDocumentEndpoint: buildCanonicalDocumentEndpoint2 } = await init_execution().then(() => exports_execution);
+          const canonical = buildCanonicalDocumentEndpoint2(context_url, intent ?? "", false);
+          if (canonical && canonical.endpoint_id === execParams.endpoint_id) {
+            console.log(`[exec] endpoint ${execParams.endpoint_id} is a canonical replay — injecting into skill`);
+            skill = { ...skill, endpoints: [canonical, ...skill.endpoints] };
+            recovered = true;
+          }
+        }
+        if (recovered) {
+          execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
+        }
+      }
       saveTrace(execResult.trace);
       if (execResult.trace.endpoint_id) {
         recordExecution(skill.skill_id, execResult.trace.endpoint_id, execResult.trace, skill).catch(() => {});
@@ -27584,7 +27638,7 @@ var init_routes = __esm(async () => {
   ]);
   BETA_API_URL = process.env.UNBROWSE_BACKEND_URL || DEFAULT_BACKEND_URL;
   TRACES_DIR = process.env.TRACES_DIR ?? join17(process.cwd(), "traces");
-  BROWSE_BROKER_MAX = Math.max(1, Number(process.env.KURI_MULTI_BROKER_MAX ?? "2"));
+  BROWSE_BROKER_MAX = Math.max(1, Number(process.env.KURI_MULTI_BROKER_MAX ?? "1"));
   BROWSE_BROKER_BASE_PORT = Number(process.env.KURI_PORT ?? "7700");
   browseSessions = new Map;
   STATS_CACHE_TTL = 5 * 60 * 1000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "3.6.0",
+  "version": "3.7.0-preview.1",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {
@@ -29,6 +29,8 @@
     "@fastify/rate-limit": "^10.3.0",
     "@cascade-fyi/splits-sdk": "^0.11.1",
     "@solana/kit": "^6.6.0",
+    "@x402/fetch": "^2.9.0",
+    "agentmail": "^0.4.18",
     "bs58": "^6.0.0",
     "cheerio": "^1.2.0",
     "dotenv": "^17.3.1",