unbrowse 3.7.0 → 3.8.0-preview.1

package/dist/cli.js

@@ -31,7 +31,7 @@ var __promiseAll = (args) => Promise.all(args);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.7.0", BUILD_GIT_SHA = "051ddb5b3add", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAiLCJnaXRfc2hhIjoiMDUxZGRiNWIzYWRkIiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUAwNTFkZGI1YjNhZGQiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTEwVDAzOjA3OjU1Ljg4MloifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "TKsUZaH-T1iT7XTgsV8ykdoZDHQj05PQkkmGr0Yf3dQ", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.8.0-preview.1", BUILD_GIT_SHA = "5b9633fd977c", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy44LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6IjViOTYzM2ZkOTc3YyIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFANWI5NjMzZmQ5NzdjIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQxNjo0NjoxMy4yMTFaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "CR3byJyWyomilicbi2WBO0kBuXqbATk49Z5ZP9B0lz8", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -1183,6 +1183,9 @@ async function getDefaultTabOn(state) {
 async function start(port, state = defaultBrokerState) {
   return startOn(state, port);
 }
+async function stop(state = defaultBrokerState) {
+  return stopOn(state);
+}
 async function getDefaultTab(state = defaultBrokerState) {
   return getDefaultTabOn(state);
 }
@@ -3375,43 +3378,59 @@ async function bootstrapAgentMailKey() {
     let currentUrl = await getCurrentUrl(tabId);
     let onDashboard = typeof currentUrl === "string" && currentUrl.includes("/dashboard");
     if (!onDashboard) {
-      log("bootstrap-agentmail", "not logged in — looking for GitHub OAuth");
-      const githubBtn = await evaluate(tabId, `(() => {
-        // Clerk renders OAuth buttons — look for GitHub
+      log("bootstrap-agentmail", "not logged in — looking for social OAuth");
+      const clickResult = await evaluate(tabId, `(() => {
+        // Clerk's social button class pattern
+        const socialBtns = document.querySelectorAll('.cl-socialButtonsBlockButton, [data-provider]');
+        for (const btn of socialBtns) {
+          const text = (btn.textContent || '').toLowerCase();
+          const provider = btn.getAttribute('data-provider') || '';
+          // Match any social provider — Google, GitHub, etc.
+          if (text || provider) {
+            btn.click();
+            return 'clicked:' + (provider || text.slice(0, 20));
+          }
+        }
+        // Fallback: look for any button with a known OAuth provider name
         const btns = document.querySelectorAll('button, a');
         for (const btn of btns) {
           const text = (btn.textContent || '').toLowerCase();
-          const ariaLabel = (btn.getAttribute('aria-label') || '').toLowerCase();
-          if (text.includes('github') || ariaLabel.includes('github') ||
-              btn.querySelector('img[alt*="github" i], svg[aria-label*="github" i]')) {
+          if (text.includes('google') || text.includes('github') || text.includes('continue with')) {
             btn.click();
-            return 'clicked';
+            return 'clicked:' + text.slice(0, 30);
           }
         }
-        // Also check for social login containers
-        const social = document.querySelector('.cl-socialButtonsBlockButton__github, [data-provider="github"]');
-        if (social) { social.click(); return 'clicked'; }
         return 'not_found';
       })()`);
-      if (githubBtn === "clicked") {
-        log("bootstrap-agentmail", "clicked GitHub OAuth — waiting for redirect");
+      const clickStr = typeof clickResult === "string" ? clickResult : String(clickResult);
+      log("bootstrap-agentmail", `social button result: ${clickStr}`);
+      if (clickStr.startsWith("clicked")) {
+        log("bootstrap-agentmail", "clicked social OAuth — waiting for redirect");
         const start2 = Date.now();
         while (Date.now() - start2 < AUTH_TIMEOUT_MS) {
           await new Promise((r) => setTimeout(r, 2000));
           currentUrl = await getCurrentUrl(tabId);
           if (typeof currentUrl === "string" && currentUrl.includes("/dashboard")) {
             onDashboard = true;
-            log("bootstrap-agentmail", "GitHub OAuth completed — on dashboard");
+            log("bootstrap-agentmail", "OAuth completed — on dashboard");
             break;
           }
         }
       }
       if (!onDashboard) {
         log("bootstrap-agentmail", "could not auto-login — manual setup needed");
+        const pageInfo = await evaluate(tabId, `JSON.stringify({
+          url: window.location.href,
+          title: document.title,
+          visible_buttons: Array.from(document.querySelectorAll('button,a'))
+            .map(e => (e.textContent || '').trim().slice(0, 30))
+            .filter(t => t)
+            .slice(0, 10),
+        })`).catch(() => "{}");
         return {
           success: false,
           method: "manual",
-          error: "Could not auto-login to AgentMail console. Sign up at https://console.agentmail.to and create an API key manually."
+          error: `Could not auto-login to AgentMail console. Page state: ${pageInfo}. Sign up at https://console.agentmail.to and create an API key manually, then: export AGENTMAIL_API_KEY=<key>`
         };
       }
     }
@@ -3483,7 +3502,11 @@ async function bootstrapAgentMailKey() {
       success: false,
       error: "Reached dashboard but could not extract API key. Create one manually at https://console.agentmail.to/dashboard/api-keys"
     };
-  } finally {}
+  } finally {
+    try {
+      await stop();
+    } catch {}
+  }
 }
 function persistApiKeyToShell(apiKey) {
   const shell = process.env.SHELL ?? "/bin/zsh";
@@ -6792,6 +6815,11 @@ async function cmdSetup(flags) {
     properties: { command: "setup" }
   });
   info("Running setup checks");
+  try {
+    await ensureRegistered({ promptForEmail: false, exitOnFailure: false });
+  } catch (err) {
+    info(`[setup] background registration issue: ${err instanceof Error ? err.message : err}`);
+  }
   const report = await runSetup({
     cwd: process.cwd(),
     opencode: normalizeSetupScope(flags.opencode),
@@ -7603,8 +7631,22 @@ async function cmdLoginAuto(args, flags) {
     }
   })();
   const { autonomousEmailLogin: autonomousEmailLogin3, isAgentMailAvailable: isAgentMailAvailable3 } = await init_agent_mail2().then(() => exports_agent_mail2);
-  if (!isAgentMailAvailable3())
-    return die("AGENTMAIL_API_KEY not set \u2014 run: export AGENTMAIL_API_KEY=<key>");
+  if (!isAgentMailAvailable3()) {
+    info(`[login-auto] AGENTMAIL_API_KEY not set \u2014 attempting auto-bootstrap via console.agentmail.to`);
+    try {
+      const { bootstrapAgentMailKey: bootstrapAgentMailKey2 } = await init_bootstrap_agentmail().then(() => exports_bootstrap_agentmail);
+      const bootstrap = await bootstrapAgentMailKey2();
+      if (bootstrap.success && bootstrap.api_key) {
+        process.env.AGENTMAIL_API_KEY = bootstrap.api_key;
+        info(`[login-auto] bootstrapped AgentMail key via ${bootstrap.method ?? "browser"}`);
+      } else {
+        return die(`AGENTMAIL_API_KEY not set and bootstrap failed: ${bootstrap.error ?? "unknown"}. Get a key at https://agentmail.to and run: export AGENTMAIL_API_KEY=<key>`);
+      }
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      return die(`AGENTMAIL_API_KEY not set and bootstrap errored: ${msg}. Get a key at https://agentmail.to and run: export AGENTMAIL_API_KEY=<key>`);
+    }
+  }
   info(`[login-auto] creating agent email for ${domain}...`);
   const session = await autonomousEmailLogin3(domain);
   info(`[login-auto] email: ${session.email}`);

package/dist/mcp.js

@@ -225,11 +225,11 @@ import { dirname, join, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.7.0";
-var BUILD_GIT_SHA = "051ddb5b3add";
+var BUILD_RELEASE_VERSION = "3.8.0-preview.1";
+var BUILD_GIT_SHA = "5b9633fd977c";
 var BUILD_CODE_HASH = "5d9ebf619c61";
-var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAiLCJnaXRfc2hhIjoiMDUxZGRiNWIzYWRkIiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUAwNTFkZGI1YjNhZGQiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTEwVDAzOjA3OjU1Ljg4MloifQ";
-var BUILD_RELEASE_MANIFEST_SIGNATURE = "TKsUZaH-T1iT7XTgsV8ykdoZDHQj05PQkkmGr0Yf3dQ";
+var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy44LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6IjViOTYzM2ZkOTc3YyIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFANWI5NjMzZmQ5NzdjIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQxNjo0NjoxMy4yMTFaIn0";
+var BUILD_RELEASE_MANIFEST_SIGNATURE = "CR3byJyWyomilicbi2WBO0kBuXqbATk49Z5ZP9B0lz8";
 var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts

package/dist/server.js

@@ -1910,6 +1910,22 @@ function deriveTemplateParamsFromContextUrl(urlTemplate, contextUrl) {
       const actualValue = actualUrl.searchParams.get(key);
       if (actualValue != null && actualValue !== "") {
         out[placeholder] = actualValue;
+        continue;
+      }
+      const byPlaceholderName = actualUrl.searchParams.get(placeholder);
+      if (byPlaceholderName != null && byPlaceholderName !== "") {
+        out[placeholder] = byPlaceholderName;
+        continue;
+      }
+      const lowerPlaceholder = placeholder.toLowerCase();
+      if (/^(q|query|search|text|keyword|keywords|term|terms)$/.test(lowerPlaceholder)) {
+        for (const alias of ["q", "query", "search", "text", "keyword", "keywords", "term"]) {
+          const v = actualUrl.searchParams.get(alias);
+          if (v != null && v !== "") {
+            out[placeholder] = v;
+            break;
+          }
+        }
       }
     }
     return out;
@@ -3827,6 +3843,13 @@ __export(exports_reverse_engineer, {
   extractAuthHeaders: () => extractAuthHeaders
 });
 import { nanoid as nanoid2 } from "nanoid";
+import { createHash } from "node:crypto";
+function stableEndpointId(method, urlTemplate) {
+  if (!method || !urlTemplate)
+    return nanoid2();
+  const hash = createHash("sha256").update(`${method}:${urlTemplate}`).digest("base64url");
+  return hash.slice(0, 21);
+}
 function extractGraphQLOperationName(url, requestBody) {
   const urlMatch = url.match(/\/graphql\/[^/]+\/(\w+)/);
   if (urlMatch)
@@ -4447,10 +4470,11 @@ function extractEndpoints(requests, wsMessages, context) {
     });
     const csrfPlan = inferCsrfPlan(req, parsedRequestBody);
     const endpointGraphqlOp = /graphql/i.test(req.url) ? extractGraphQLOperationName(req.url, req.request_body) : undefined;
+    const computedUrlTemplate = qTemplateStr ? `${pathTemplate}${pathTemplate.includes("?") ? "&" : "?"}${qTemplateStr}` : pathTemplate;
     let endpoint = {
-      endpoint_id: nanoid2(),
+      endpoint_id: stableEndpointId(req.method, computedUrlTemplate),
       method: req.method,
-      url_template: qTemplateStr ? `${pathTemplate}${pathTemplate.includes("?") ? "&" : "?"}${qTemplateStr}` : pathTemplate,
+      url_template: computedUrlTemplate,
       description: buildEndpointDescription(req, sampleRequest, sampleResponse),
       headers_template: sanitizeHeaders(req.request_headers),
       query: sanitizedQParams,
@@ -4541,7 +4565,7 @@ function extractEndpoints(requests, wsMessages, context) {
         response_schema = inferSchema(jsonSamples);
       }
       const endpoint = {
-        endpoint_id: nanoid2(),
+        endpoint_id: stableEndpointId("WS", wsUrl),
         method: "WS",
         url_template: wsUrl,
         idempotency: "safe",
@@ -7096,10 +7120,10 @@ var init_capture = __esm(async () => {
 });
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.7.0", BUILD_GIT_SHA = "051ddb5b3add", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAiLCJnaXRfc2hhIjoiMDUxZGRiNWIzYWRkIiwiY29kZV9oYXNoIjoiNWQ5ZWJmNjE5YzYxIiwidHJhY2VfdmVyc2lvbiI6IjVkOWViZjYxOWM2MUAwNTFkZGI1YjNhZGQiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTEwVDAzOjA3OjU1Ljg4MloifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "TKsUZaH-T1iT7XTgsV8ykdoZDHQj05PQkkmGr0Yf3dQ", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.8.0-preview.1", BUILD_GIT_SHA = "5b9633fd977c", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy44LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6IjViOTYzM2ZkOTc3YyIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFANWI5NjMzZmQ5NzdjIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQxNjo0NjoxMy4yMTFaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "CR3byJyWyomilicbi2WBO0kBuXqbATk49Z5ZP9B0lz8", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
-import { createHash } from "crypto";
+import { createHash as createHash2 } from "crypto";
 import { existsSync as existsSync4, readFileSync as readFileSync2, readdirSync } from "fs";
 import { dirname, join as join3, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
@@ -7116,7 +7140,7 @@ function collectTsFiles(dir) {
   return results;
 }
 function hashFiles(srcDir, files) {
-  const hash = createHash("sha256");
+  const hash = createHash2("sha256");
   for (const file of files) {
     hash.update(file.slice(srcDir.length));
     hash.update(readFileSync2(file, "utf-8"));
@@ -7156,7 +7180,7 @@ function computeCodeHash() {
   } catch {}
   const pkgVersion = getPackageVersion();
   if (pkgVersion !== "unknown") {
-    return createHash("sha256").update(`package:${pkgVersion}`).digest("hex").slice(0, 12);
+    return createHash2("sha256").update(`package:${pkgVersion}`).digest("hex").slice(0, 12);
   }
   return "compiled";
 }
@@ -7210,13 +7234,13 @@ var init_version = __esm(() => {
 });
 
 // ../../src/payments/cascade.ts
-import { createHash as createHash2 } from "crypto";
+import { createHash as createHash3 } from "crypto";
 import bs58 from "bs58";
 function payableContributors(skill) {
   return (skill.contributors ?? []).filter((c) => !!c.wallet_address?.trim());
 }
 function cascadeLabel(skillId) {
-  const digest = createHash2("sha256").update(skillId).digest("hex");
+  const digest = createHash3("sha256").update(skillId).digest("hex");
   return `ubr-${digest.slice(0, 23)}`;
 }
 function decodeSecretKey(raw) {
@@ -7610,7 +7634,7 @@ __export(exports_client2, {
 import { readFileSync as readFileSync4, writeFileSync as writeFileSync3, existsSync as existsSync7, mkdirSync as mkdirSync4, readdirSync as readdirSync2, unlinkSync } from "fs";
 import { join as join6 } from "path";
 import { homedir as homedir4, hostname, release as osRelease } from "os";
-import { randomBytes as randomBytes2, createHash as createHash3 } from "crypto";
+import { randomBytes as randomBytes2, createHash as createHash4 } from "crypto";
 import { createInterface } from "readline";
 import { execSync } from "child_process";
 function buildReleaseAttestationHeaders(manifestBase64, signature) {
@@ -7887,7 +7911,7 @@ function getApiKey() {
 function hashApiKey(key) {
   if (!key || key === "local-only")
     return "";
-  return createHash3("sha256").update(key).digest("hex");
+  return createHash4("sha256").update(key).digest("hex");
 }
 function getAgentId() {
   const config = loadConfig();
@@ -9069,7 +9093,7 @@ var init_publish_admission = __esm(() => {
 });
 
 // ../../src/telemetry.ts
-import { createHash as createHash4 } from "node:crypto";
+import { createHash as createHash5 } from "node:crypto";
 import { existsSync as existsSync8, mkdirSync as mkdirSync5, writeFileSync as writeFileSync4 } from "node:fs";
 import { join as join7 } from "node:path";
 function getTraceDir() {
@@ -9079,7 +9103,7 @@ function isTracingEnabled() {
   return process.env.UNBROWSE_DISABLE_TRACES !== "1";
 }
 function hashValue(value) {
-  return createHash4("sha256").update(value).digest("hex").slice(0, 16);
+  return createHash5("sha256").update(value).digest("hex").slice(0, 16);
 }
 function isSensitiveName(name) {
   return SENSITIVE_PATTERNS.some((re) => re.test(name));
@@ -9097,7 +9121,7 @@ function anonymizeUrl(url) {
 }
 function hashResponseBody(result) {
   const str = typeof result === "string" ? result : JSON.stringify(result ?? "");
-  return createHash4("sha256").update(str).digest("hex").slice(0, 32);
+  return createHash5("sha256").update(str).digest("hex").slice(0, 32);
 }
 function classifyFailure(error) {
   if (!error)
@@ -12457,7 +12481,12 @@ function cleanDOM(html) {
   const $ = cheerio.load(html);
   for (const tag of STRIP_TAGS) {
     if (tag === "script") {
-      $("script").not('[type="application/ld+json"]').remove();
+      $("script").each((_, el) => {
+        const type = $(el).attr("type") ?? "";
+        if (type !== "application/ld+json") {
+          $(el).remove();
+        }
+      });
     } else {
       $(tag).remove();
     }
@@ -15495,7 +15524,12 @@ __export(exports_execution, {
   buildCanonicalDocumentEndpoint: () => buildCanonicalDocumentEndpoint
 });
 import { nanoid as nanoid6 } from "nanoid";
-import { createHash as createHash5 } from "node:crypto";
+import { createHash as createHash6 } from "node:crypto";
+function stableEndpointId2(method, urlTemplate) {
+  if (!method || !urlTemplate)
+    return nanoid6();
+  return createHash6("sha256").update(`${method}:${urlTemplate}`).digest("base64url").slice(0, 21);
+}
 function stampTrace(trace) {
   trace.trace_version = TRACE_VERSION;
   return trace;
@@ -16039,10 +16073,11 @@ function buildPageArtifactCapture(url, intent, html, authRequired = false) {
   const searchForms = detectSearchForms(html);
   const validSearchForm = searchForms.find((spec) => isStructuredSearchForm(spec));
   const response_schema = inferSchema([extracted.data]);
+  const computedTemplate = templatizeQueryParams(url);
   const endpoint = {
-    endpoint_id: nanoid6(),
+    endpoint_id: stableEndpointId2("GET", computedTemplate),
     method: "GET",
-    url_template: templatizeQueryParams(url),
+    url_template: computedTemplate,
     idempotency: "safe",
     verification_status: "verified",
     reliability_score: extracted.confidence,
@@ -16090,7 +16125,7 @@ function buildCanonicalDocumentEndpoint(url, intent, authRequired = false) {
   const replayTemplate = deriveStructuredDataReplayTemplate(url);
   if (replayUrl === url && replayTemplate === url)
     return;
-  const canonicalId = createHash5("sha1").update(replayTemplate !== url ? replayTemplate : replayUrl).digest("base64url").slice(0, 21);
+  const canonicalId = createHash6("sha1").update(replayTemplate !== url ? replayTemplate : replayUrl).digest("base64url").slice(0, 21);
   const endpoint = {
     endpoint_id: canonicalId,
     method: "GET",
@@ -16270,6 +16305,65 @@ async function trySeedPublicDocumentFetchSkill(skill, url, intent, targetDomain,
     redirect: "follow"
   });
   const html = await response.text();
+  const contentType = (response.headers.get("content-type") || "").toLowerCase();
+  if (response.ok && (contentType.includes("application/json") || contentType.includes("text/json"))) {
+    try {
+      const parsed = JSON.parse(html);
+      const urlObj = new URL(response.url || url);
+      const pathTemplate = `${urlObj.origin}${urlObj.pathname}`;
+      const responseSchema = inferSchema([parsed]);
+      const endpoint = {
+        endpoint_id: stableEndpointId2("GET", pathTemplate),
+        method: "GET",
+        url_template: pathTemplate,
+        idempotency: "safe",
+        verification_status: "verified",
+        reliability_score: 0.95,
+        description: `Direct JSON API for ${intent}`,
+        response_schema: responseSchema
+      };
+      endpoint.semantic = inferEndpointSemantic(endpoint, {
+        sampleResponse: parsed,
+        observedAt: new Date().toISOString(),
+        sampleRequestUrl: url
+      });
+      const domain2 = getRegistrableDomain(targetDomain);
+      const existingSkill2 = findExistingSkillForDomain(domain2, intent);
+      const localEndpoints2 = await prepareLearnedEndpoints(existingSkill2 ? mergeEndpoints(existingSkill2.endpoints, [endpoint]) : [endpoint], intent, domain2);
+      const localDraft2 = {
+        skill_id: existingSkill2?.skill_id ?? nanoid6(),
+        version: "1.0.0",
+        schema_version: "1",
+        lifecycle: "active",
+        execution_type: "http",
+        created_at: existingSkill2?.created_at ?? new Date().toISOString(),
+        updated_at: new Date().toISOString(),
+        name: domain2,
+        intent_signature: intent,
+        domain: domain2,
+        description: `API skill for ${domain2}`,
+        owner_type: "agent",
+        endpoints: localEndpoints2,
+        operation_graph: buildSkillOperationGraph(localEndpoints2),
+        intents: [intent]
+      };
+      try {
+        cachePublishedSkill(localDraft2);
+      } catch {}
+      return {
+        trace: stampTrace({
+          trace_id: nanoid6(),
+          skill_id: localDraft2.skill_id,
+          endpoint_id: endpoint.endpoint_id,
+          started_at: new Date().toISOString(),
+          completed_at: new Date().toISOString(),
+          success: true,
+          status_code: response.status
+        }),
+        result: parsed
+      };
+    } catch {}
+  }
   if (!isHtml(html) || isSpaShell(html))
     return;
   const built = buildPageArtifactCapture(response.url || url, intent, html, usedStoredAuth);
@@ -16595,7 +16689,7 @@ async function executeBrowserCapture(skill, params, options) {
         epUrl = `${route.url}?${qStr}`;
       }
       endpoints.push({
-        endpoint_id: nanoid6(),
+        endpoint_id: stableEndpointId2("GET", epUrl),
         method: "GET",
         url_template: epUrl,
         query: epQuery,
@@ -16745,6 +16839,56 @@ async function executeBrowserCapture(skill, params, options) {
         }
       };
     }
+    const antiBotSignal = (() => {
+      const html = captured.html ?? "";
+      if (!html)
+        return "empty_capture";
+      const lower = html.toLowerCase();
+      const titleMatch = lower.match(/<title[^>]*>([^<]{0,200})<\/title>/);
+      const title = titleMatch ? titleMatch[1].trim() : "";
+      const blockTitles = [
+        "robot check",
+        "access denied",
+        "just a moment",
+        "attention required",
+        "please verify",
+        "security check",
+        "are you a robot"
+      ];
+      for (const marker of blockTitles) {
+        if (title.includes(marker))
+          return `anti_bot_title:${marker}`;
+      }
+      const authTitles = ["log in", "sign in", "log in to", "sign in to"];
+      if (authTitles.some((t) => title === t || title.startsWith(t + " ")) || title === "login") {
+        return `auth_wall_title:${title}`;
+      }
+      if (lower.includes("g-recaptcha") || lower.includes("hcaptcha.com") || lower.includes("cf-challenge-form")) {
+        return "captcha_present";
+      }
+      if (html.length < 5000 && !lower.includes("<script")) {
+        return `tiny_body:${html.length}b`;
+      }
+      return null;
+    })();
+    if (antiBotSignal) {
+      const trace3 = stampTrace({
+        trace_id: traceId,
+        skill_id: skill.skill_id,
+        endpoint_id: "browser-capture",
+        started_at: startedAt,
+        completed_at: new Date().toISOString(),
+        success: false,
+        error: "browser_block"
+      });
+      return {
+        trace: trace3,
+        result: {
+          error: "browser_block",
+          message: `Browser capture detected a block signal at ${url}: ${antiBotSignal}. The site served a degraded/challenge page to the headless browser. Real-browser cookies via autoExtract may be required.`
+        }
+      };
+    }
     const trace2 = stampTrace({
       trace_id: traceId,
       skill_id: skill.skill_id,
@@ -19117,10 +19261,10 @@ var init_timing_economics = __esm(() => {
 });
 
 // ../../src/routing-telemetry.ts
-import { createHash as createHash6 } from "node:crypto";
+import { createHash as createHash7 } from "node:crypto";
 import { nanoid as nanoid8 } from "nanoid";
 function stableHash(value) {
-  return createHash6("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 24);
+  return createHash7("sha256").update(JSON.stringify(value)).digest("hex").slice(0, 24);
 }
 function sanitizeScalar(value) {
   if (value == null)
@@ -19474,7 +19618,7 @@ function applyVerificationResults(skill, verification) {
 import { nanoid as nanoid9 } from "nanoid";
 import { existsSync as existsSync13, writeFileSync as writeFileSync8, readFileSync as readFileSync8, mkdirSync as mkdirSync9, readdirSync as readdirSync6 } from "node:fs";
 import { dirname as dirname2, join as join12 } from "node:path";
-import { createHash as createHash7 } from "node:crypto";
+import { createHash as createHash8 } from "node:crypto";
 function summarizeSchema(schema, maxDepth = 3) {
   function walk(s, depth) {
     if (depth <= 0)
@@ -19602,7 +19746,7 @@ function scopedResolveCacheKeys(scope, key) {
   return scope === "global" ? [scopedCacheKey("global", key)] : [scopedCacheKey(scope, key), scopedCacheKey("global", key)];
 }
 function snapshotPathForCacheKey(cacheKey) {
-  const digest = createHash7("sha1").update(cacheKey).digest("hex");
+  const digest = createHash8("sha1").update(cacheKey).digest("hex");
   return join12(SKILL_SNAPSHOT_DIR, `${digest}.json`);
 }
 function writeSkillSnapshot(cacheKey, skill) {
@@ -19953,7 +20097,7 @@ function buildLocalCanonicalReplaySkill(intent, contextUrl) {
   const domain = new URL(contextUrl).hostname.replace(/^www\./, "");
   const now = new Date().toISOString();
   const skill = {
-    skill_id: `canonical-${createHash7("sha1").update(contextUrl).digest("hex").slice(0, 12)}`,
+    skill_id: `canonical-${createHash8("sha1").update(contextUrl).digest("hex").slice(0, 12)}`,
     version: "1.0.0",
     schema_version: "1",
     name: `Canonical replay for ${domain}`,
@@ -22237,35 +22381,58 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
       }
     }
   }
-  if (context?.url && (/\.(json|xml)(\?|$)|\/api\/|\/v\d+\//.test(context.url) || /[?&]format=(j\d*|json)\b/i.test(context.url) || /^https?:\/\/api\./i.test(context.url))) {
+  if (context?.url) {
     try {
       const directRes = await fetch(context.url, {
-        headers: { Accept: "application/json", "User-Agent": "unbrowse/1.0" },
-        signal: AbortSignal.timeout(5000),
+        headers: { Accept: "application/json, text/html;q=0.5", "User-Agent": "unbrowse/1.0" },
+        signal: AbortSignal.timeout(15000),
         redirect: "follow"
       });
       const ct = directRes.headers.get("content-type") ?? "";
-      if (directRes.ok && (ct.includes("json") || ct.includes("+json"))) {
-        const data = await directRes.json();
-        const trace2 = {
-          trace_id: nanoid9(),
-          skill_id: "direct-fetch",
-          endpoint_id: "direct-fetch",
-          started_at: new Date().toISOString(),
-          completed_at: new Date().toISOString(),
-          success: true
-        };
-        const t = finalize("direct-fetch", data, "direct-fetch", undefined, trace2);
-        console.log(`[direct-fetch] ${context.url} returned JSON directly — skipping browser`);
-        return {
-          result: data,
-          trace: trace2,
-          source: "direct-fetch",
-          skill: undefined,
-          timing: t
-        };
+      const ctSaysJson = ct.includes("application/json") || ct.includes("+json") || ct.includes("text/json");
+      if (directRes.ok) {
+        let data = undefined;
+        if (ctSaysJson) {
+          data = await directRes.json();
+        } else {
+          const bodyText = await directRes.text();
+          if (bodyText.length < 2000000) {
+            const trimmed = bodyText.trimStart();
+            if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
+              try {
+                const parsed = JSON.parse(bodyText);
+                if (parsed !== null && typeof parsed === "object") {
+                  data = parsed;
+                  console.log(`[direct-fetch] ${context.url} body-sniff hit: ct="${ct}" but body parses as JSON`);
+                }
+              } catch {}
+            }
+          }
+        }
+        if (data !== undefined) {
+          const trace2 = {
+            trace_id: nanoid9(),
+            skill_id: "direct-fetch",
+            endpoint_id: "direct-fetch",
+            started_at: new Date().toISOString(),
+            completed_at: new Date().toISOString(),
+            success: true
+          };
+          const t = finalize("direct-fetch", data, "direct-fetch", undefined, trace2);
+          console.log(`[direct-fetch] ${context.url} returned JSON directly — skipping browser`);
+          return {
+            result: data,
+            trace: trace2,
+            source: "direct-fetch",
+            skill: undefined,
+            timing: t
+          };
+        }
       }
-    } catch {}
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.log(`[direct-fetch] ${context.url} skipped: ${msg.slice(0, 100)}`);
+    }
   }
   if (process.env.UNBROWSE_LOCAL_ONLY === "1" && !forceCapture) {
     return buildNoCachedMatch();
@@ -22424,6 +22591,32 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
     throw error;
   }
   timing.execute_ms = Date.now() - te0;
+  const captureErrCheck = result?.error;
+  if (captureErrCheck === "connection_failed" || captureErrCheck === "capture_failed") {
+    console.warn(`[capture] ${captureErrCheck} detected — restarting Kuri and retrying once`);
+    try {
+      const kuri = await Promise.resolve().then(() => (init_client(), exports_client));
+      await kuri.stop().catch(() => {});
+      await new Promise((r) => setTimeout(r, 500));
+    } catch {}
+    try {
+      const retryCaptureSkill = await getOrCreateBrowserCaptureSkill();
+      const retryOut = await withAbortableOpTimeout("live_capture_retry", LIVE_CAPTURE_TIMEOUT_MS, (signal) => executeSkill(retryCaptureSkill, { ...params, url: context.url, intent }, undefined, {
+        ...options,
+        intent,
+        contextUrl: context?.url,
+        signal
+      }));
+      if (retryOut.trace.success || !retryOut.result?.error) {
+        trace = retryOut.trace;
+        result = retryOut.result;
+        learned_skill = retryOut.learned_skill;
+        console.log(`[capture] retry after Kuri restart succeeded`);
+      }
+    } catch (retryErr) {
+      console.warn(`[capture] retry failed: ${retryErr instanceof Error ? retryErr.message : retryErr}`);
+    }
+  }
   const captureResult = result;
   const authRecommended = captureResult?.auth_recommended === true;
   const directDomCaptureResult = trace.success && trace.endpoint_id !== "browser-capture" && !!result && typeof result === "object" && "_extraction" in result;
@@ -24207,7 +24400,13 @@ var init_browse_session = __esm(() => {
 
 // ../../src/api/browse-index.ts
 import { nanoid as nanoid10 } from "nanoid";
+import { createHash as createHash9 } from "node:crypto";
 import { readFileSync as readFileSync12 } from "node:fs";
+function stableEndpointId3(method, urlTemplate) {
+  if (!method || !urlTemplate)
+    return nanoid10();
+  return createHash9("sha256").update(`${method}:${urlTemplate}`).digest("base64url").slice(0, 21);
+}
 function normalizeBrowseUrl(url, baseUrl) {
   if (!url)
     return url;
@@ -24385,7 +24584,7 @@ async function cacheBrowseRequests(params) {
       return { domain, indexed: false, mode: "none", skill: null };
     const urlTemplate = templatizeQueryParams2(sessionUrl);
     const endpoint = {
-      endpoint_id: nanoid10(),
+      endpoint_id: stableEndpointId3("GET", urlTemplate),
       method: "GET",
       url_template: urlTemplate,
       idempotency: "safe",
@@ -26437,11 +26636,18 @@ async function registerRoutes(app) {
   app.addHook("onRequest", async (req, reply) => {
     if (req.url === "/health" || req.url === "/v1/stats" || req.url.startsWith("/v1/settings"))
       return;
-    const key = getApiKey();
+    let key = getApiKey();
+    if (!key) {
+      try {
+        const { waitForBackgroundRegistration: waitForBackgroundRegistration2 } = await Promise.resolve().then(() => (init_client2(), exports_client2));
+        await waitForBackgroundRegistration2(1e4);
+        key = getApiKey();
+      } catch {}
+    }
     if (!key) {
       return reply.code(401).send({
         error: "api_key_required",
-        message: "No API key configured. Restart the server to auto-register, or run: bash scripts/setup.sh",
+        message: "No API key configured. Run `unbrowse setup` to register, or set UNBROWSE_API_KEY manually.",
         docs_url: "https://unbrowse.ai"
       });
     }
@@ -26691,7 +26897,28 @@ async function registerRoutes(app) {
       ...context_url && typeof params?.url !== "string" ? { url: context_url } : {}
     };
     try {
-      const execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
+      let execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
+      if (!execResult.trace.success && execResult.result?.error === "endpoint_not_found" && typeof execParams.endpoint_id === "string") {
+        let recovered = false;
+        const freshSkill = await getSkill2(skill_id, clientScope);
+        if (freshSkill && freshSkill.endpoints.some((e) => e.endpoint_id === execParams.endpoint_id)) {
+          console.log(`[exec] endpoint ${execParams.endpoint_id} found in fresh marketplace skill — retrying`);
+          skill = freshSkill;
+          recovered = true;
+        }
+        if (!recovered && context_url) {
+          const { buildCanonicalDocumentEndpoint: buildCanonicalDocumentEndpoint2 } = await init_execution().then(() => exports_execution);
+          const canonical = buildCanonicalDocumentEndpoint2(context_url, intent ?? "", false);
+          if (canonical && canonical.endpoint_id === execParams.endpoint_id) {
+            console.log(`[exec] endpoint ${execParams.endpoint_id} is a canonical replay — injecting into skill`);
+            skill = { ...skill, endpoints: [canonical, ...skill.endpoints] };
+            recovered = true;
+          }
+        }
+        if (recovered) {
+          execResult = await executeSkill(skill, execParams, projection, { confirm_unsafe, confirm_third_party_terms, dry_run, skip_robots_check, intent, contextUrl: context_url, client_scope: clientScope });
+        }
+      }
       saveTrace(execResult.trace);
       if (execResult.trace.endpoint_id) {
         recordExecution(skill.skill_id, execResult.trace.endpoint_id, execResult.trace, skill).catch(() => {});

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "3.7.0",
+  "version": "3.8.0-preview.1",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {