unbrowse 3.2.1 → 3.2.2

package/dist/cli.js

@@ -31,7 +31,7 @@ var __promiseAll = (args) => Promise.all(args);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.2.1", BUILD_GIT_SHA = "150cce0d751e", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjEiLCJnaXRfc2hhIjoiMTUwY2NlMGQ3NTFlIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0AxNTBjY2UwZDc1MWUiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDA3OjM0OjU3LjQ1NFoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "2iiYVQS4ow2XkpkyCm072lmrjIIGvkAPjkO1s_LI_Do", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.2.2", BUILD_GIT_SHA = "150cce0d751e", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjIiLCJnaXRfc2hhIjoiMTUwY2NlMGQ3NTFlIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0AxNTBjY2UwZDc1MWUiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDA4OjQ2OjM5LjIwNVoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "ZS7C10DGuwyn1m02shMHyz6cN5UbTHfa8yqnkELg_L8", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash } from "crypto";

package/dist/mcp.js

@@ -225,11 +225,11 @@ import { dirname, join, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.2.1";
+var BUILD_RELEASE_VERSION = "3.2.2";
 var BUILD_GIT_SHA = "150cce0d751e";
 var BUILD_CODE_HASH = "1488fc1d92b7";
-var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjEiLCJnaXRfc2hhIjoiMTUwY2NlMGQ3NTFlIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0AxNTBjY2UwZDc1MWUiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDA3OjM0OjU3LjQ1NFoifQ";
-var BUILD_RELEASE_MANIFEST_SIGNATURE = "2iiYVQS4ow2XkpkyCm072lmrjIIGvkAPjkO1s_LI_Do";
+var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjIiLCJnaXRfc2hhIjoiMTUwY2NlMGQ3NTFlIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0AxNTBjY2UwZDc1MWUiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDA4OjQ2OjM5LjIwNVoifQ";
+var BUILD_RELEASE_MANIFEST_SIGNATURE = "ZS7C10DGuwyn1m02shMHyz6cN5UbTHfa8yqnkELg_L8";
 var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts

package/dist/server.js

@@ -4210,7 +4210,7 @@ function extractEndpoints(requests, wsMessages, context) {
           return "";
         }
       })();
-      const isApiUrl = /\/(api|graphql)\b/i.test(urlPath) || /\.(json)(\?|$)/.test(req.url);
+      const isApiUrl = /\/(api|graphql|youtubei|__ssr_data__)\b/i.test(urlPath) || /\.(json)(\?|$)/.test(req.url);
       let graphqlOpName;
       if (/graphql/i.test(req.url)) {
         if (req.request_body) {
@@ -4946,7 +4946,7 @@ var init_reverse_engineer = __esm(() => {
   SKIP_HOSTS = /(cloudflare\.com|google-analytics\.com|doubleclick\.net|gstatic\.com|accounts\.google\.com|login\.microsoftonline\.com|auth0\.com|cognito-idp\.|appleid\.apple\.com|github\.com\/login|facebook\.com\/login|protechts\.net|demdex\.net|litms|platform-telemetry|datadoghq\.com|fullstory\.com|launchdarkly\.com|intercom\.io|privy\.io|mypinata\.cloud|sentry\.io|segment\.io|amplitude\.com|mixpanel\.com|hotjar\.com|clarity\.ms|googletagmanager\.com|walletconnect\.com|imagedelivery\.net|cloudflareinsights\.com)/i;
   SKIP_TELEMETRY_HOSTS = /(waa-pa\.|signaler-pa\.|appsgrowthpromo-pa\.|ogads-pa\.|peoplestackwebexperiments-pa\.)/i;
   SKIP_TELEMETRY_PATHS = /\/(log|logging|telemetry|analytics|beacon|ping|heartbeat|metrics)(\/|$)/i;
-  RPC_HINTS = /(\/$rpc\/|\/rpc\/|graphql|trending|search|feed|results|batchexecute|\/api\/)/i;
+  RPC_HINTS = /(\/$rpc\/|\/rpc\/|graphql|trending|search|feed|results|batchexecute|\/api\/|youtubei|__ssr_data__)/i;
   ALLOWED_METHODS = new Set(["GET", "POST", "PUT", "PATCH", "DELETE"]);
   STRIP_HEADERS = new Set([
     "cookie",
@@ -5024,7 +5024,7 @@ var init_reverse_engineer = __esm(() => {
     "adsize",
     "lineitemid"
   ]);
-  ON_DOMAIN_NOISE = /\/(recaptcha|captcha|update-recaptcha|csrf|consent|data-protection|badge|drawer|header-action|geolocation|onboarding|wana\/bids|prebid|bids\/request|ads\/|pixel|beacon|collect|impression|click-tracking|heartbeat|webConfig|config\.json|manifest\.json|service-worker|sw\.js|favicon|robots\.txt|sitemap|opensearch|partial\/[a-zA-Z]+\/mod-|logging|csp-report|gen_204|generate_204|sodar|__|devvit-|user-drawer|action-item)/i;
+  ON_DOMAIN_NOISE = /\/(recaptcha|captcha|update-recaptcha|csrf|consent|data-protection|badge|drawer|header-action|geolocation|onboarding|wana\/bids|prebid|bids\/request|ads\/|pixel|beacon|collect|impression|click-tracking|heartbeat|webConfig|config\.json|manifest\.json|service-worker|sw\.js|favicon|robots\.txt|sitemap|opensearch|partial\/[a-zA-Z]+\/mod-|logging|csp-report|gen_204|generate_204|sodar|__webpack|__next|devvit-|user-drawer|action-item)/i;
 });
 
 // ../../src/runtime/browser-access.ts
@@ -5952,7 +5952,7 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
       if (status < 200 || status >= 400)
         continue;
       const ct = (entry.response?.headers ?? []).find((h) => h.name.toLowerCase() === "content-type")?.value ?? "";
-      if (!HAR_REPLAY_CT2.test(ct) && !harUrl.includes("/api/") && !harUrl.includes("_search") && !harUrl.includes("graphql"))
+      if (!HAR_REPLAY_CT2.test(ct) && !harUrl.includes("/api/") && !harUrl.includes("_search") && !harUrl.includes("graphql") && !harUrl.includes("youtubei"))
         continue;
       if (harReplayCount >= 20)
         break;
@@ -5987,6 +5987,70 @@ async function captureSession(url, authHeaders, cookies, intent, options) {
       }
       html = await phase("getPageHtml", () => getPageHtml(tabId));
     } catch {}
+    const SSR_DATA_EXTRACTORS = [
+      { name: "ytmusic", script: `(function(){try{var d=ytcfg.get('YTMUSIC_INITIAL_DATA');if(!d||!d.length)return null;var out={};d.forEach(function(x){if(x.path&&x.data)out[x.path]=x.data});return JSON.stringify(out)}catch(e){return null}})()` },
+      { name: "youtube", script: `(function(){try{return typeof ytInitialData!=='undefined'?JSON.stringify(ytInitialData):null}catch(e){return null}})()` },
+      { name: "nextjs", script: `(function(){try{return window.__NEXT_DATA__?JSON.stringify(window.__NEXT_DATA__):null}catch(e){return null}})()` },
+      { name: "nuxt", script: `(function(){try{return window.__NUXT__?JSON.stringify(window.__NUXT__):null}catch(e){return null}})()` }
+    ];
+    let embeddedDataCount = 0;
+    for (const extractor of SSR_DATA_EXTRACTORS) {
+      try {
+        const raw = await phase(`ssr:${extractor.name}`, () => evaluate(tabId, extractor.script));
+        if (typeof raw !== "string" || !raw || raw === "null")
+          continue;
+        if (extractor.name === "ytmusic") {
+          const paths = JSON.parse(raw);
+          for (const [path4, data] of Object.entries(paths)) {
+            if (!data || typeof data !== "object")
+              continue;
+            const cleaned = { ...data };
+            delete cleaned.responseContext;
+            delete cleaned.trackingParams;
+            delete cleaned.header;
+            delete cleaned.background;
+            let bodyStr = JSON.stringify(cleaned);
+            if (bodyStr.length > 1e4) {
+              bodyStr = bodyStr.substring(0, 1e4) + '"}]}';
+              try {
+                JSON.parse(bodyStr);
+              } catch {
+                bodyStr = JSON.stringify(cleaned).substring(0, 1e4);
+              }
+            }
+            if (bodyStr.length < 100)
+              continue;
+            const origin = new URL(url).origin;
+            const contextParams = new URL(url).searchParams;
+            const queryStr = path4.includes("search") && contextParams.toString() ? `?${contextParams.toString()}&prettyPrint=false` : "?prettyPrint=false";
+            const syntheticUrl = `${origin}/youtubei/v1${path4}${queryStr}`;
+            responseBodies.set(syntheticUrl, bodyStr);
+            harEntries.push({
+              startedDateTime: new Date().toISOString(),
+              request: { method: "POST", url: syntheticUrl, headers: [{ name: "content-type", value: "application/json" }], postData: { text: "{}" } },
+              response: { status: 200, headers: [{ name: "content-type", value: "application/json" }], content: { text: bodyStr, mimeType: "application/json" } }
+            });
+            embeddedDataCount++;
+            log("capture", `ssr:${extractor.name} extracted ${path4} (${bodyStr.length}B)`);
+          }
+        } else {
+          if (raw.length < 100)
+            continue;
+          const syntheticUrl = `${new URL(url).origin}/__ssr_data__/${extractor.name}`;
+          responseBodies.set(syntheticUrl, raw);
+          harEntries.push({
+            startedDateTime: new Date().toISOString(),
+            request: { method: "GET", url: syntheticUrl, headers: [] },
+            response: { status: 200, headers: [{ name: "content-type", value: "application/json" }], content: { text: raw, mimeType: "application/json" } }
+          });
+          embeddedDataCount++;
+          log("capture", `ssr:${extractor.name} extracted (${raw.length}B)`);
+        }
+      } catch {}
+    }
+    if (embeddedDataCount > 0) {
+      log("capture", `embedded SSR data: ${embeddedDataCount} synthetic endpoints injected`);
+    }
     const requests = mergePassiveCaptureData(intercepted, harEntries, extensionEntries, responseBodies, performanceUrls);
     log("capture", `tracked ${harEntries.length} HAR, ${intercepted.length} intercepted, ${extensionEntries.length} extension, ${responseBodies.size} bodies → ${requests.length} merged`);
     const rawCookies = await phase("extractCookies", () => extractCookiesFromPage(tabId, url));
@@ -6385,7 +6449,8 @@ var MAX_CONCURRENT_TABS = 3, activeTabs = 0, waitQueue, activeTabRegistry, inter
       var isData = ct.indexOf('json') !== -1 || ct.indexOf('application/x-protobuf') !== -1 ||
                    ct.indexOf('text/plain') !== -1 ||
                    url.indexOf('batchexecute') !== -1 || url.indexOf('/api/') !== -1 ||
-                   url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1;
+                   url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1 ||
+                   url.indexOf('youtubei') !== -1;
       if (!isJs && !isData) return response;
       if (/\\.(css|woff2?|png|jpg|svg|ico)(\\?|$)/.test(url)) return response;
       var clone = response.clone();
@@ -6435,7 +6500,8 @@ var MAX_CONCURRENT_TABS = 3, activeTabs = 0, waitQueue, activeTabRegistry, inter
       var isData = ct.indexOf('json') !== -1 || ct.indexOf('application/x-protobuf') !== -1 ||
                    ct.indexOf('text/plain') !== -1 ||
                    url.indexOf('batchexecute') !== -1 || url.indexOf('/api/') !== -1 ||
-                   url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1;
+                   url.indexOf('graphql') !== -1 || url.indexOf('voyager') !== -1 ||
+                   url.indexOf('youtubei') !== -1;
       if (!isJs && !isData) return;
       if (/\\.(css|woff2?|png|jpg|svg|ico)(\\?|$)/.test(url)) return;
       var respBody = xhr.responseText || '';
@@ -6478,7 +6544,7 @@ var init_capture = __esm(() => {
 });
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.2.1", BUILD_GIT_SHA = "150cce0d751e", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjEiLCJnaXRfc2hhIjoiMTUwY2NlMGQ3NTFlIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0AxNTBjY2UwZDc1MWUiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDA3OjM0OjU3LjQ1NFoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "2iiYVQS4ow2XkpkyCm072lmrjIIGvkAPjkO1s_LI_Do", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.2.2", BUILD_GIT_SHA = "150cce0d751e", BUILD_CODE_HASH = "1488fc1d92b7", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy4yLjIiLCJnaXRfc2hhIjoiMTUwY2NlMGQ3NTFlIiwiY29kZV9oYXNoIjoiMTQ4OGZjMWQ5MmI3IiwidHJhY2VfdmVyc2lvbiI6IjE0ODhmYzFkOTJiN0AxNTBjY2UwZDc1MWUiLCJpc3N1ZWRfYXQiOiIyMDI2LTA0LTA2VDA4OjQ2OjM5LjIwNVoifQ", BUILD_RELEASE_MANIFEST_SIGNATURE = "ZS7C10DGuwyn1m02shMHyz6cN5UbTHfa8yqnkELg_L8", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -16709,6 +16775,17 @@ function rankEndpoints(endpoints, intent, skillDomain, contextUrl) {
       }
       score += matches * 100;
     }
+    if (rawTokens.length > 0 && pathname) {
+      const pathLower = pathname.toLowerCase();
+      const pathSegs = pathLower.split("/").filter(Boolean);
+      for (const token of rawTokens) {
+        const stemmed = stem(token);
+        if (pathSegs.some((seg) => seg === stemmed || seg === token || seg.includes(token))) {
+          score += 150;
+          break;
+        }
+      }
+    }
     if (ep.dom_extraction)
       score += 25;
     if (descriptionMeta.needs_review && ep.dom_extraction)
@@ -18568,6 +18645,13 @@ function isCachedSkillRelevantForIntent(skill, intent, contextUrl) {
     const hasStructuredSearchEndpoint = candidateSkill.endpoints.some((endpoint) => endpointHasSearchBindings(endpoint) && (!!endpoint.dom_extraction || !!endpoint.response_schema) && endpointMatchesContextOrigin(endpoint, contextUrl) && endpointMatchesExplicitSearchContext(endpoint, contextUrl));
     if (hasStructuredSearchEndpoint)
       return true;
+    if (top && top.score >= 0) {
+      try {
+        const topPath = new URL(top.endpoint.url_template).pathname.toLowerCase();
+        if (/\/(search|find|query|browse|explore)\b/.test(topPath))
+          return true;
+      } catch {}
+    }
     if (collectExplicitSearchContextBindingKeys(contextUrl).size > 0)
       return false;
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "3.2.1",
+  "version": "3.2.2",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {