unbrowse 3.7.0-preview.2 → 3.7.0-preview.4

package/dist/cli.js

@@ -31,7 +31,7 @@ var __promiseAll = (args) => Promise.all(args);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.7.0-preview.2", BUILD_GIT_SHA = "c3b7d2a563b0", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4yIiwiZ2l0X3NoYSI6ImMzYjdkMmE1NjNiMCIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYzNiN2QyYTU2M2IwIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNTo0OTo0OS45OTBaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "Y_s0r-BxEwdCh5t91laYK58bdu_st2vyzaQdPhhepGY", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.7.0-preview.4", BUILD_GIT_SHA = "989697af2dfd", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy40IiwiZ2l0X3NoYSI6Ijk4OTY5N2FmMmRmZCIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAOTg5Njk3YWYyZGZkIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwODoxMTowOS40MzRaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "F2X5UWb9JvjD7-NVKq9uOeZ3Alwpjhxs5as_-YGmjl4", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -3375,43 +3375,59 @@ async function bootstrapAgentMailKey() {
     let currentUrl = await getCurrentUrl(tabId);
     let onDashboard = typeof currentUrl === "string" && currentUrl.includes("/dashboard");
     if (!onDashboard) {
-      log("bootstrap-agentmail", "not logged in — looking for GitHub OAuth");
-      const githubBtn = await evaluate(tabId, `(() => {
-        // Clerk renders OAuth buttons — look for GitHub
+      log("bootstrap-agentmail", "not logged in — looking for social OAuth");
+      const clickResult = await evaluate(tabId, `(() => {
+        // Clerk's social button class pattern
+        const socialBtns = document.querySelectorAll('.cl-socialButtonsBlockButton, [data-provider]');
+        for (const btn of socialBtns) {
+          const text = (btn.textContent || '').toLowerCase();
+          const provider = btn.getAttribute('data-provider') || '';
+          // Match any social provider — Google, GitHub, etc.
+          if (text || provider) {
+            btn.click();
+            return 'clicked:' + (provider || text.slice(0, 20));
+          }
+        }
+        // Fallback: look for any button with a known OAuth provider name
         const btns = document.querySelectorAll('button, a');
         for (const btn of btns) {
           const text = (btn.textContent || '').toLowerCase();
-          const ariaLabel = (btn.getAttribute('aria-label') || '').toLowerCase();
-          if (text.includes('github') || ariaLabel.includes('github') ||
-              btn.querySelector('img[alt*="github" i], svg[aria-label*="github" i]')) {
+          if (text.includes('google') || text.includes('github') || text.includes('continue with')) {
             btn.click();
-            return 'clicked';
+            return 'clicked:' + text.slice(0, 30);
           }
         }
-        // Also check for social login containers
-        const social = document.querySelector('.cl-socialButtonsBlockButton__github, [data-provider="github"]');
-        if (social) { social.click(); return 'clicked'; }
         return 'not_found';
       })()`);
-      if (githubBtn === "clicked") {
-        log("bootstrap-agentmail", "clicked GitHub OAuth — waiting for redirect");
+      const clickStr = typeof clickResult === "string" ? clickResult : String(clickResult);
+      log("bootstrap-agentmail", `social button result: ${clickStr}`);
+      if (clickStr.startsWith("clicked")) {
+        log("bootstrap-agentmail", "clicked social OAuth — waiting for redirect");
         const start2 = Date.now();
         while (Date.now() - start2 < AUTH_TIMEOUT_MS) {
           await new Promise((r) => setTimeout(r, 2000));
           currentUrl = await getCurrentUrl(tabId);
           if (typeof currentUrl === "string" && currentUrl.includes("/dashboard")) {
             onDashboard = true;
-            log("bootstrap-agentmail", "GitHub OAuth completed — on dashboard");
+            log("bootstrap-agentmail", "OAuth completed — on dashboard");
             break;
           }
         }
       }
       if (!onDashboard) {
         log("bootstrap-agentmail", "could not auto-login — manual setup needed");
+        const pageInfo = await evaluate(tabId, `JSON.stringify({
+          url: window.location.href,
+          title: document.title,
+          visible_buttons: Array.from(document.querySelectorAll('button,a'))
+            .map(e => (e.textContent || '').trim().slice(0, 30))
+            .filter(t => t)
+            .slice(0, 10),
+        })`).catch(() => "{}");
         return {
           success: false,
           method: "manual",
-          error: "Could not auto-login to AgentMail console. Sign up at https://console.agentmail.to and create an API key manually."
+          error: `Could not auto-login to AgentMail console. Page state: ${pageInfo}. Sign up at https://console.agentmail.to and create an API key manually, then: export AGENTMAIL_API_KEY=<key>`
         };
       }
     }
@@ -7603,8 +7619,22 @@ async function cmdLoginAuto(args, flags) {
     }
   })();
   const { autonomousEmailLogin: autonomousEmailLogin3, isAgentMailAvailable: isAgentMailAvailable3 } = await init_agent_mail2().then(() => exports_agent_mail2);
-  if (!isAgentMailAvailable3())
-    return die("AGENTMAIL_API_KEY not set \u2014 run: export AGENTMAIL_API_KEY=<key>");
+  if (!isAgentMailAvailable3()) {
+    info(`[login-auto] AGENTMAIL_API_KEY not set \u2014 attempting auto-bootstrap via console.agentmail.to`);
+    try {
+      const { bootstrapAgentMailKey: bootstrapAgentMailKey2 } = await init_bootstrap_agentmail().then(() => exports_bootstrap_agentmail);
+      const bootstrap = await bootstrapAgentMailKey2();
+      if (bootstrap.success && bootstrap.api_key) {
+        process.env.AGENTMAIL_API_KEY = bootstrap.api_key;
+        info(`[login-auto] bootstrapped AgentMail key via ${bootstrap.method ?? "browser"}`);
+      } else {
+        return die(`AGENTMAIL_API_KEY not set and bootstrap failed: ${bootstrap.error ?? "unknown"}. Get a key at https://agentmail.to and run: export AGENTMAIL_API_KEY=<key>`);
+      }
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      return die(`AGENTMAIL_API_KEY not set and bootstrap errored: ${msg}. Get a key at https://agentmail.to and run: export AGENTMAIL_API_KEY=<key>`);
+    }
+  }
   info(`[login-auto] creating agent email for ${domain}...`);
   const session = await autonomousEmailLogin3(domain);
   info(`[login-auto] email: ${session.email}`);

package/dist/mcp.js

@@ -225,11 +225,11 @@ import { dirname, join, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.7.0-preview.2";
-var BUILD_GIT_SHA = "c3b7d2a563b0";
+var BUILD_RELEASE_VERSION = "3.7.0-preview.4";
+var BUILD_GIT_SHA = "989697af2dfd";
 var BUILD_CODE_HASH = "5d9ebf619c61";
-var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4yIiwiZ2l0X3NoYSI6ImMzYjdkMmE1NjNiMCIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYzNiN2QyYTU2M2IwIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNTo0OTo0OS45OTBaIn0";
-var BUILD_RELEASE_MANIFEST_SIGNATURE = "Y_s0r-BxEwdCh5t91laYK58bdu_st2vyzaQdPhhepGY";
+var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy40IiwiZ2l0X3NoYSI6Ijk4OTY5N2FmMmRmZCIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAOTg5Njk3YWYyZGZkIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwODoxMTowOS40MzRaIn0";
+var BUILD_RELEASE_MANIFEST_SIGNATURE = "F2X5UWb9JvjD7-NVKq9uOeZ3Alwpjhxs5as_-YGmjl4";
 var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts

package/dist/server.js

@@ -7120,7 +7120,7 @@ var init_capture = __esm(async () => {
 });
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.7.0-preview.2", BUILD_GIT_SHA = "c3b7d2a563b0", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4yIiwiZ2l0X3NoYSI6ImMzYjdkMmE1NjNiMCIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYzNiN2QyYTU2M2IwIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNTo0OTo0OS45OTBaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "Y_s0r-BxEwdCh5t91laYK58bdu_st2vyzaQdPhhepGY", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.7.0-preview.4", BUILD_GIT_SHA = "989697af2dfd", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy40IiwiZ2l0X3NoYSI6Ijk4OTY5N2FmMmRmZCIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAOTg5Njk3YWYyZGZkIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwODoxMTowOS40MzRaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "F2X5UWb9JvjD7-NVKq9uOeZ3Alwpjhxs5as_-YGmjl4", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash as createHash2 } from "crypto";
@@ -12481,7 +12481,12 @@ function cleanDOM(html) {
   const $ = cheerio.load(html);
   for (const tag of STRIP_TAGS) {
     if (tag === "script") {
-      $("script").not('[type="application/ld+json"]').remove();
+      $("script").each((_, el) => {
+        const type = $(el).attr("type") ?? "";
+        if (type !== "application/ld+json") {
+          $(el).remove();
+        }
+      });
     } else {
       $(tag).remove();
     }
@@ -16300,6 +16305,65 @@ async function trySeedPublicDocumentFetchSkill(skill, url, intent, targetDomain,
     redirect: "follow"
   });
   const html = await response.text();
+  const contentType = (response.headers.get("content-type") || "").toLowerCase();
+  if (response.ok && (contentType.includes("application/json") || contentType.includes("text/json"))) {
+    try {
+      const parsed = JSON.parse(html);
+      const urlObj = new URL(response.url || url);
+      const pathTemplate = `${urlObj.origin}${urlObj.pathname}`;
+      const responseSchema = inferSchema([parsed]);
+      const endpoint = {
+        endpoint_id: stableEndpointId2("GET", pathTemplate),
+        method: "GET",
+        url_template: pathTemplate,
+        idempotency: "safe",
+        verification_status: "verified",
+        reliability_score: 0.95,
+        description: `Direct JSON API for ${intent}`,
+        response_schema: responseSchema
+      };
+      endpoint.semantic = inferEndpointSemantic(endpoint, {
+        sampleResponse: parsed,
+        observedAt: new Date().toISOString(),
+        sampleRequestUrl: url
+      });
+      const domain2 = getRegistrableDomain(targetDomain);
+      const existingSkill2 = findExistingSkillForDomain(domain2, intent);
+      const localEndpoints2 = await prepareLearnedEndpoints(existingSkill2 ? mergeEndpoints(existingSkill2.endpoints, [endpoint]) : [endpoint], intent, domain2);
+      const localDraft2 = {
+        skill_id: existingSkill2?.skill_id ?? nanoid6(),
+        version: "1.0.0",
+        schema_version: "1",
+        lifecycle: "active",
+        execution_type: "http",
+        created_at: existingSkill2?.created_at ?? new Date().toISOString(),
+        updated_at: new Date().toISOString(),
+        name: domain2,
+        intent_signature: intent,
+        domain: domain2,
+        description: `API skill for ${domain2}`,
+        owner_type: "agent",
+        endpoints: localEndpoints2,
+        operation_graph: buildSkillOperationGraph(localEndpoints2),
+        intents: [intent]
+      };
+      try {
+        cachePublishedSkill(localDraft2);
+      } catch {}
+      return {
+        trace: stampTrace({
+          trace_id: nanoid6(),
+          skill_id: localDraft2.skill_id,
+          endpoint_id: endpoint.endpoint_id,
+          started_at: new Date().toISOString(),
+          completed_at: new Date().toISOString(),
+          success: true,
+          status_code: response.status
+        }),
+        result: parsed
+      };
+    } catch {}
+  }
   if (!isHtml(html) || isSpaShell(html))
     return;
   const built = buildPageArtifactCapture(response.url || url, intent, html, usedStoredAuth);
@@ -22267,15 +22331,15 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
       }
     }
   }
-  if (context?.url && (/\.(json|xml)(\?|$)|\/api\/|\/v\d+\//.test(context.url) || /[?&]format=(j\d*|json)\b/i.test(context.url) || /^https?:\/\/api\./i.test(context.url))) {
+  if (context?.url) {
     try {
       const directRes = await fetch(context.url, {
-        headers: { Accept: "application/json", "User-Agent": "unbrowse/1.0" },
-        signal: AbortSignal.timeout(5000),
+        headers: { Accept: "application/json, text/html;q=0.5", "User-Agent": "unbrowse/1.0" },
+        signal: AbortSignal.timeout(15000),
         redirect: "follow"
       });
       const ct = directRes.headers.get("content-type") ?? "";
-      if (directRes.ok && (ct.includes("json") || ct.includes("+json"))) {
+      if (directRes.ok && (ct.includes("application/json") || ct.includes("+json") || ct.includes("text/json"))) {
         const data = await directRes.json();
         const trace2 = {
           trace_id: nanoid9(),
@@ -22295,7 +22359,10 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
           timing: t
         };
       }
-    } catch {}
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.log(`[direct-fetch] ${context.url} skipped: ${msg.slice(0, 100)}`);
+    }
   }
   if (process.env.UNBROWSE_LOCAL_ONLY === "1" && !forceCapture) {
     return buildNoCachedMatch();
@@ -22454,6 +22521,32 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
     throw error;
   }
   timing.execute_ms = Date.now() - te0;
+  const captureErrCheck = result?.error;
+  if (captureErrCheck === "connection_failed" || captureErrCheck === "capture_failed") {
+    console.warn(`[capture] ${captureErrCheck} detected — restarting Kuri and retrying once`);
+    try {
+      const kuri = await Promise.resolve().then(() => (init_client(), exports_client));
+      await kuri.stop().catch(() => {});
+      await new Promise((r) => setTimeout(r, 500));
+    } catch {}
+    try {
+      const retryCaptureSkill = await getOrCreateBrowserCaptureSkill();
+      const retryOut = await withAbortableOpTimeout("live_capture_retry", LIVE_CAPTURE_TIMEOUT_MS, (signal) => executeSkill(retryCaptureSkill, { ...params, url: context.url, intent }, undefined, {
+        ...options,
+        intent,
+        contextUrl: context?.url,
+        signal
+      }));
+      if (retryOut.trace.success || !retryOut.result?.error) {
+        trace = retryOut.trace;
+        result = retryOut.result;
+        learned_skill = retryOut.learned_skill;
+        console.log(`[capture] retry after Kuri restart succeeded`);
+      }
+    } catch (retryErr) {
+      console.warn(`[capture] retry failed: ${retryErr instanceof Error ? retryErr.message : retryErr}`);
+    }
+  }
   const captureResult = result;
   const authRecommended = captureResult?.auth_recommended === true;
   const directDomCaptureResult = trace.success && trace.endpoint_id !== "browser-capture" && !!result && typeof result === "object" && "_extraction" in result;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "3.7.0-preview.2",
+  "version": "3.7.0-preview.4",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {