unbrowse 3.7.0-preview.1 → 3.7.0-preview.3

package/dist/cli.js

@@ -31,7 +31,7 @@ var __promiseAll = (args) => Promise.all(args);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.7.0-preview.1", BUILD_GIT_SHA = "b3ecde042c51", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6ImIzZWNkZTA0MmM1MSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYjNlY2RlMDQyYzUxIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNToxNDowNy4xMDRaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "BQOPfluuXHVkQx0LryOJrfGDa-ctFVdDBcNhh53WHQQ", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.7.0-preview.3", BUILD_GIT_SHA = "652904c236e5", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4zIiwiZ2l0X3NoYSI6IjY1MjkwNGMyMzZlNSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFANjUyOTA0YzIzNmU1IiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNzoyNzozNS44ODZaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "IXw8XfBaII9eVNJCmgvwRyesbKiWJVNMurzf5P2mRb0", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash } from "crypto";

package/dist/mcp.js

@@ -225,11 +225,11 @@ import { dirname, join, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.7.0-preview.1";
-var BUILD_GIT_SHA = "b3ecde042c51";
+var BUILD_RELEASE_VERSION = "3.7.0-preview.3";
+var BUILD_GIT_SHA = "652904c236e5";
 var BUILD_CODE_HASH = "5d9ebf619c61";
-var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6ImIzZWNkZTA0MmM1MSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYjNlY2RlMDQyYzUxIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNToxNDowNy4xMDRaIn0";
-var BUILD_RELEASE_MANIFEST_SIGNATURE = "BQOPfluuXHVkQx0LryOJrfGDa-ctFVdDBcNhh53WHQQ";
+var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4zIiwiZ2l0X3NoYSI6IjY1MjkwNGMyMzZlNSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFANjUyOTA0YzIzNmU1IiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNzoyNzozNS44ODZaIn0";
+var BUILD_RELEASE_MANIFEST_SIGNATURE = "IXw8XfBaII9eVNJCmgvwRyesbKiWJVNMurzf5P2mRb0";
 var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts

package/dist/server.js

@@ -1910,6 +1910,22 @@ function deriveTemplateParamsFromContextUrl(urlTemplate, contextUrl) {
       const actualValue = actualUrl.searchParams.get(key);
       if (actualValue != null && actualValue !== "") {
         out[placeholder] = actualValue;
+        continue;
+      }
+      const byPlaceholderName = actualUrl.searchParams.get(placeholder);
+      if (byPlaceholderName != null && byPlaceholderName !== "") {
+        out[placeholder] = byPlaceholderName;
+        continue;
+      }
+      const lowerPlaceholder = placeholder.toLowerCase();
+      if (/^(q|query|search|text|keyword|keywords|term|terms)$/.test(lowerPlaceholder)) {
+        for (const alias of ["q", "query", "search", "text", "keyword", "keywords", "term"]) {
+          const v = actualUrl.searchParams.get(alias);
+          if (v != null && v !== "") {
+            out[placeholder] = v;
+            break;
+          }
+        }
       }
     }
     return out;
@@ -7104,7 +7120,7 @@ var init_capture = __esm(async () => {
 });
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.7.0-preview.1", BUILD_GIT_SHA = "b3ecde042c51", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4xIiwiZ2l0X3NoYSI6ImIzZWNkZTA0MmM1MSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAYjNlY2RlMDQyYzUxIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNToxNDowNy4xMDRaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "BQOPfluuXHVkQx0LryOJrfGDa-ctFVdDBcNhh53WHQQ", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.7.0-preview.3", BUILD_GIT_SHA = "652904c236e5", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy43LjAtcHJldmlldy4zIiwiZ2l0X3NoYSI6IjY1MjkwNGMyMzZlNSIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFANjUyOTA0YzIzNmU1IiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQwNzoyNzozNS44ODZaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "IXw8XfBaII9eVNJCmgvwRyesbKiWJVNMurzf5P2mRb0", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash as createHash2 } from "crypto";
@@ -12465,7 +12481,12 @@ function cleanDOM(html) {
   const $ = cheerio.load(html);
   for (const tag of STRIP_TAGS) {
     if (tag === "script") {
-      $("script").not('[type="application/ld+json"]').remove();
+      $("script").each((_, el) => {
+        const type = $(el).attr("type") ?? "";
+        if (type !== "application/ld+json") {
+          $(el).remove();
+        }
+      });
     } else {
       $(tag).remove();
     }
@@ -16284,6 +16305,65 @@ async function trySeedPublicDocumentFetchSkill(skill, url, intent, targetDomain,
     redirect: "follow"
   });
   const html = await response.text();
+  const contentType = (response.headers.get("content-type") || "").toLowerCase();
+  if (response.ok && (contentType.includes("application/json") || contentType.includes("text/json"))) {
+    try {
+      const parsed = JSON.parse(html);
+      const urlObj = new URL(response.url || url);
+      const pathTemplate = `${urlObj.origin}${urlObj.pathname}`;
+      const responseSchema = inferSchema([parsed]);
+      const endpoint = {
+        endpoint_id: stableEndpointId2("GET", pathTemplate),
+        method: "GET",
+        url_template: pathTemplate,
+        idempotency: "safe",
+        verification_status: "verified",
+        reliability_score: 0.95,
+        description: `Direct JSON API for ${intent}`,
+        response_schema: responseSchema
+      };
+      endpoint.semantic = inferEndpointSemantic(endpoint, {
+        sampleResponse: parsed,
+        observedAt: new Date().toISOString(),
+        sampleRequestUrl: url
+      });
+      const domain2 = getRegistrableDomain(targetDomain);
+      const existingSkill2 = findExistingSkillForDomain(domain2, intent);
+      const localEndpoints2 = await prepareLearnedEndpoints(existingSkill2 ? mergeEndpoints(existingSkill2.endpoints, [endpoint]) : [endpoint], intent, domain2);
+      const localDraft2 = {
+        skill_id: existingSkill2?.skill_id ?? nanoid6(),
+        version: "1.0.0",
+        schema_version: "1",
+        lifecycle: "active",
+        execution_type: "http",
+        created_at: existingSkill2?.created_at ?? new Date().toISOString(),
+        updated_at: new Date().toISOString(),
+        name: domain2,
+        intent_signature: intent,
+        domain: domain2,
+        description: `API skill for ${domain2}`,
+        owner_type: "agent",
+        endpoints: localEndpoints2,
+        operation_graph: buildSkillOperationGraph(localEndpoints2),
+        intents: [intent]
+      };
+      try {
+        cachePublishedSkill(localDraft2);
+      } catch {}
+      return {
+        trace: stampTrace({
+          trace_id: nanoid6(),
+          skill_id: localDraft2.skill_id,
+          endpoint_id: endpoint.endpoint_id,
+          started_at: new Date().toISOString(),
+          completed_at: new Date().toISOString(),
+          success: true,
+          status_code: response.status
+        }),
+        result: parsed
+      };
+    } catch {}
+  }
   if (!isHtml(html) || isSpaShell(html))
     return;
   const built = buildPageArtifactCapture(response.url || url, intent, html, usedStoredAuth);
@@ -22251,15 +22331,15 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
       }
     }
   }
-  if (context?.url && (/\.(json|xml)(\?|$)|\/api\/|\/v\d+\//.test(context.url) || /[?&]format=(j\d*|json)\b/i.test(context.url) || /^https?:\/\/api\./i.test(context.url))) {
+  if (context?.url) {
     try {
       const directRes = await fetch(context.url, {
-        headers: { Accept: "application/json", "User-Agent": "unbrowse/1.0" },
-        signal: AbortSignal.timeout(5000),
+        headers: { Accept: "application/json, text/html;q=0.5", "User-Agent": "unbrowse/1.0" },
+        signal: AbortSignal.timeout(15000),
         redirect: "follow"
       });
       const ct = directRes.headers.get("content-type") ?? "";
-      if (directRes.ok && (ct.includes("json") || ct.includes("+json"))) {
+      if (directRes.ok && (ct.includes("application/json") || ct.includes("+json") || ct.includes("text/json"))) {
         const data = await directRes.json();
         const trace2 = {
           trace_id: nanoid9(),
@@ -22279,7 +22359,10 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
           timing: t
         };
       }
-    } catch {}
+    } catch (err) {
+      const msg = err instanceof Error ? err.message : String(err);
+      console.log(`[direct-fetch] ${context.url} skipped: ${msg.slice(0, 100)}`);
+    }
   }
   if (process.env.UNBROWSE_LOCAL_ONLY === "1" && !forceCapture) {
     return buildNoCachedMatch();
@@ -22438,6 +22521,32 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
     throw error;
   }
   timing.execute_ms = Date.now() - te0;
+  const captureErrCheck = result?.error;
+  if (captureErrCheck === "connection_failed" || captureErrCheck === "capture_failed") {
+    console.warn(`[capture] ${captureErrCheck} detected — restarting Kuri and retrying once`);
+    try {
+      const kuri = await Promise.resolve().then(() => (init_client(), exports_client));
+      await kuri.stop().catch(() => {});
+      await new Promise((r) => setTimeout(r, 500));
+    } catch {}
+    try {
+      const retryCaptureSkill = await getOrCreateBrowserCaptureSkill();
+      const retryOut = await withAbortableOpTimeout("live_capture_retry", LIVE_CAPTURE_TIMEOUT_MS, (signal) => executeSkill(retryCaptureSkill, { ...params, url: context.url, intent }, undefined, {
+        ...options,
+        intent,
+        contextUrl: context?.url,
+        signal
+      }));
+      if (retryOut.trace.success || !retryOut.result?.error) {
+        trace = retryOut.trace;
+        result = retryOut.result;
+        learned_skill = retryOut.learned_skill;
+        console.log(`[capture] retry after Kuri restart succeeded`);
+      }
+    } catch (retryErr) {
+      console.warn(`[capture] retry failed: ${retryErr instanceof Error ? retryErr.message : retryErr}`);
+    }
+  }
   const captureResult = result;
   const authRecommended = captureResult?.auth_recommended === true;
   const directDomCaptureResult = trace.success && trace.endpoint_id !== "browser-capture" && !!result && typeof result === "object" && "_extraction" in result;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "3.7.0-preview.1",
+  "version": "3.7.0-preview.3",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {