unbrowse 3.8.0-preview.2 → 3.8.0-preview.3

package/dist/cli.js

@@ -31,7 +31,7 @@ var __promiseAll = (args) => Promise.all(args);
 var __require = /* @__PURE__ */ createRequire(import.meta.url);
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.8.0-preview.2", BUILD_GIT_SHA = "efefee1f85df", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy44LjAtcHJldmlldy4yIiwiZ2l0X3NoYSI6ImVmZWZlZTFmODVkZiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAZWZlZmVlMWY4NWRmIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQxNzowOTo0OS44NzNaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "nVliy0ydfucv7W56hI5CiuHzYJ92ve4oJqgzV1NFASg", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.8.0-preview.3", BUILD_GIT_SHA = "dea0516c7186", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy44LjAtcHJldmlldy4zIiwiZ2l0X3NoYSI6ImRlYTA1MTZjNzE4NiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAZGVhMDUxNmM3MTg2IiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMVQwMjo1OTo0NC4zMTVaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "4DMqMBhc7BFMz0Q9wxIWg0Rib1Dv1aVs9waZQIpXK4Q", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash } from "crypto";
@@ -3007,6 +3007,7 @@ var init_execution = __esm(async () => {
   init_bundle_scanner();
   init_token_resolver();
   init_marketplace();
+  init_publish_admission();
   init_transform();
   init_drift();
   init_client();

package/dist/mcp.js

@@ -225,11 +225,11 @@ import { dirname, join, parse } from "path";
 import { fileURLToPath as fileURLToPath2 } from "url";
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.8.0-preview.2";
-var BUILD_GIT_SHA = "efefee1f85df";
+var BUILD_RELEASE_VERSION = "3.8.0-preview.3";
+var BUILD_GIT_SHA = "dea0516c7186";
 var BUILD_CODE_HASH = "5d9ebf619c61";
-var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy44LjAtcHJldmlldy4yIiwiZ2l0X3NoYSI6ImVmZWZlZTFmODVkZiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAZWZlZmVlMWY4NWRmIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQxNzowOTo0OS44NzNaIn0";
-var BUILD_RELEASE_MANIFEST_SIGNATURE = "nVliy0ydfucv7W56hI5CiuHzYJ92ve4oJqgzV1NFASg";
+var BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy44LjAtcHJldmlldy4zIiwiZ2l0X3NoYSI6ImRlYTA1MTZjNzE4NiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAZGVhMDUxNmM3MTg2IiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMVQwMjo1OTo0NC4zMTVaIn0";
+var BUILD_RELEASE_MANIFEST_SIGNATURE = "4DMqMBhc7BFMz0Q9wxIWg0Rib1Dv1aVs9waZQIpXK4Q";
 var BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts

package/dist/server.js

@@ -4234,6 +4234,17 @@ function getIntentEntityRules(kind) {
   }
 }
 function isSemanticallyAdmissibleResponse(req, sampleResponse, sampleRequest, context) {
+  if (/\/graphql(\/|$|\?)/i.test(req.url)) {
+    const noiseOpRe = /globalnav|sidenav|navdash|topnav|nav_|notification|notif_|preloadstate|tracking|telemetry|pingback|heartbeat|presence|rightrail|gno_|viewertracking|metadata$|\bmetadata\b/i;
+    const opName = extractGraphQLOperationName(req.url, req.request_body) ?? "";
+    const opMatchesNoise = !!opName && noiseOpRe.test(opName);
+    const urlMatchesNoise = noiseOpRe.test(req.url);
+    const bodyMatchesNoise = typeof req.request_body === "string" && noiseOpRe.test(req.request_body.slice(0, 500));
+    if (opMatchesNoise || urlMatchesNoise || bodyMatchesNoise) {
+      return { ok: false, reason: "graphql_noise_operation" };
+    }
+    return { ok: true, reason: "semantic_graphql_bypass" };
+  }
   const kind = inferIntentEntityKind(context?.intent);
   const action2 = inferIntentActionKind(context?.intent);
   if (!kind) {
@@ -4338,7 +4349,7 @@ function scoreRequest(req) {
   }
   return score;
 }
-function extractEndpoints(requests, wsMessages, context) {
+function extractEndpoints(requests, wsMessages, context, traceSink) {
   const seen = new Set;
   const endpoints = [];
   const traceRows = [];
@@ -4362,6 +4373,16 @@ function extractEndpoints(requests, wsMessages, context) {
       continue;
     }
     if (!hasAdmissibleParsedBody(req.response_body)) {
+      const body = req.response_body;
+      if (typeof body === "string" && body.length > 20) {
+        const trimmed = body.trimStart().slice(0, 200).toLowerCase();
+        const looksCss = /^[.#@a-z][a-z0-9_\-]*\s*\{|^body\s*\{|^@media|^@import|^@font-face/.test(trimmed);
+        const looksJs = /^function\s|^var\s|^let\s|^const\s|^import\s|^\(function|^\/\*|^\!function/.test(trimmed) && !trimmed.includes('{"');
+        if (looksCss || looksJs) {
+          traceRows.push({ url: req.url, method: req.method, score, kept: false, reason: "body_not_json_or_html" });
+          continue;
+        }
+      }
       const urlPath = (() => {
         try {
           return new URL(req.url).pathname;
@@ -4369,7 +4390,7 @@ function extractEndpoints(requests, wsMessages, context) {
           return "";
         }
       })();
-      const isApiUrl = /\/(api|graphql|youtubei|__ssr_data__)\b/i.test(urlPath) || /\.(json)(\?|$)/.test(req.url);
+      const isApiUrl = /\/(api|graphql|youtubei|__ssr_data__|_next\/data|xhr|ajax|rest)\b/i.test(urlPath) || /\/v\d+\//i.test(urlPath) || /\/async[-_]/i.test(urlPath) || /[-_](state|data|feed|timeline|search|list|results)(\?|$|\/)/i.test(urlPath) || /\.(json)(\?|$)/.test(req.url);
       let graphqlOpName;
       if (/graphql/i.test(req.url)) {
         graphqlOpName = extractGraphQLOperationName(req.url, req.request_body);
@@ -4393,8 +4414,21 @@ function extractEndpoints(requests, wsMessages, context) {
         const reqHost = new URL(req.url).hostname;
         const reqDomain = getRegistrableDomain(reqHost);
         if (!affinityDomains.has(reqDomain)) {
-          traceRows.push({ url: req.url, method: req.method, score, kept: false, reason: "domain_mismatch" });
-          continue;
+          const reqBrand = reqDomain.split(".")[0] ?? "";
+          let siblingOk = false;
+          if (reqBrand.length >= 4) {
+            for (const a of affinityDomains) {
+              const aBrand = a.split(".")[0] ?? "";
+              if (aBrand.length >= 4 && (reqBrand.startsWith(aBrand) || aBrand.startsWith(reqBrand))) {
+                siblingOk = true;
+                break;
+              }
+            }
+          }
+          if (!siblingOk) {
+            traceRows.push({ url: req.url, method: req.method, score, kept: false, reason: "domain_mismatch" });
+            continue;
+          }
         }
       } catch {
         traceRows.push({ url: req.url, method: req.method, score, kept: false, reason: "bad_url" });
@@ -4599,6 +4633,8 @@ function extractEndpoints(requests, wsMessages, context) {
       resource_kind: endpoint.semantic?.resource_kind
     }))
   });
+  if (traceSink)
+    traceSink.rows = traceRows;
   return endpoints;
 }
 function isApiLike(req) {
@@ -7120,7 +7156,7 @@ var init_capture = __esm(async () => {
 });
 
 // ../../src/build-info.generated.ts
-var BUILD_RELEASE_VERSION = "3.8.0-preview.2", BUILD_GIT_SHA = "efefee1f85df", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy44LjAtcHJldmlldy4yIiwiZ2l0X3NoYSI6ImVmZWZlZTFmODVkZiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAZWZlZmVlMWY4NWRmIiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMFQxNzowOTo0OS44NzNaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "nVliy0ydfucv7W56hI5CiuHzYJ92ve4oJqgzV1NFASg", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
+var BUILD_RELEASE_VERSION = "3.8.0-preview.3", BUILD_GIT_SHA = "dea0516c7186", BUILD_CODE_HASH = "5d9ebf619c61", BUILD_RELEASE_MANIFEST_BASE64 = "eyJzY2hlbWFfdmVyc2lvbiI6MSwicmVsZWFzZV92ZXJzaW9uIjoiMy44LjAtcHJldmlldy4zIiwiZ2l0X3NoYSI6ImRlYTA1MTZjNzE4NiIsImNvZGVfaGFzaCI6IjVkOWViZjYxOWM2MSIsInRyYWNlX3ZlcnNpb24iOiI1ZDllYmY2MTljNjFAZGVhMDUxNmM3MTg2IiwiaXNzdWVkX2F0IjoiMjAyNi0wNC0xMVQwMjo1OTo0NC4zMTVaIn0", BUILD_RELEASE_MANIFEST_SIGNATURE = "4DMqMBhc7BFMz0Q9wxIWg0Rib1Dv1aVs9waZQIpXK4Q", BUILD_DEFAULT_BACKEND_URL = "https://beta-api.unbrowse.ai";
 
 // ../../src/version.ts
 import { createHash as createHash2 } from "crypto";
@@ -8793,6 +8829,7 @@ function freshReasonCounts() {
     noise: 0,
     fragile_graphql: 0,
     no_durable_signal: 0,
+    dom_fallback_only: 0,
     family_dedup: 0,
     over_limit: 0
   };
@@ -8812,6 +8849,9 @@ function looksOpaqueIdentifier(value) {
   return hasAlpha && hasDigit;
 }
 function isCanonicalDocumentReplay(endpoint) {
+  const extractionMethod = endpoint.dom_extraction?.extraction_method ?? "";
+  if (extractionMethod.startsWith("spa-"))
+    return false;
   if (/captured page artifact/i.test(endpoint.description ?? ""))
     return true;
   if (!endpoint.trigger_url)
@@ -8963,6 +9003,27 @@ function selectMarketplacePublishEndpoints(skill, options = {}) {
       score: scoreEndpoint(endpoint)
     });
   }
+  const hasRealEndpoint = candidates.some((c) => !isCanonicalDocumentReplay(c.endpoint));
+  if (!hasRealEndpoint) {
+    reasons.dom_fallback_only += candidates.length;
+    return {
+      endpoints: [],
+      stats: {
+        total: skill.endpoints?.length ?? 0,
+        kept: 0,
+        by_reason: reasons
+      }
+    };
+  }
+  const filteredCandidates = candidates.filter((c) => {
+    if (isCanonicalDocumentReplay(c.endpoint)) {
+      reasons.dom_fallback_only += 1;
+      return false;
+    }
+    return true;
+  });
+  candidates.length = 0;
+  candidates.push(...filteredCandidates);
   candidates.sort((a, b) => {
     if (b.score !== a.score)
       return b.score - a.score;
@@ -12408,6 +12469,161 @@ function extractFlashNoticeSpecial(html, intent) {
     selector: buildReplaySelector(flash)
   }];
 }
+function sliceBalancedObject(src, startIdx) {
+  const first = src.indexOf("{", startIdx);
+  if (first < 0)
+    return null;
+  let depth = 0;
+  let inString = false;
+  let stringChar = "";
+  let escaped = false;
+  for (let i = first;i < src.length; i++) {
+    const c = src[i];
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+    if (inString) {
+      if (c === "\\") {
+        escaped = true;
+        continue;
+      }
+      if (c === stringChar) {
+        inString = false;
+      }
+      continue;
+    }
+    if (c === '"' || c === "'" || c === "`") {
+      inString = true;
+      stringChar = c;
+      continue;
+    }
+    if (c === "{")
+      depth += 1;
+    else if (c === "}") {
+      depth -= 1;
+      if (depth === 0)
+        return src.substring(first, i + 1);
+    }
+  }
+  return null;
+}
+function sliceBalancedAny(src, startIdx) {
+  const open = src[startIdx];
+  if (open !== "{" && open !== "[")
+    return null;
+  const closeCh = open === "{" ? "}" : "]";
+  let depth = 0;
+  let inString = false;
+  let stringChar = "";
+  let escaped = false;
+  for (let i = startIdx;i < src.length; i++) {
+    const c = src[i];
+    if (escaped) {
+      escaped = false;
+      continue;
+    }
+    if (inString) {
+      if (c === "\\") {
+        escaped = true;
+        continue;
+      }
+      if (c === stringChar) {
+        inString = false;
+      }
+      continue;
+    }
+    if (c === '"' || c === "'" || c === "`") {
+      inString = true;
+      stringChar = c;
+      continue;
+    }
+    if (c === open)
+      depth += 1;
+    else if (c === closeCh) {
+      depth -= 1;
+      if (depth === 0)
+        return src.substring(startIdx, i + 1);
+    }
+  }
+  return null;
+}
+function findWindowAssignmentPayload(html, varName) {
+  const assignRe = new RegExp(String.raw`window\.${varName}\s*=\s*(\{)`, "i");
+  const m = assignRe.exec(html);
+  if (!m)
+    return null;
+  const startIdx = m.index + m[0].length - 1;
+  return sliceBalancedObject(html, startIdx);
+}
+function unwrapInfiniteQuery(data) {
+  if (!data || typeof data !== "object")
+    return [];
+  const d = data;
+  const pages = d.pages;
+  if (!Array.isArray(pages) || pages.length === 0)
+    return [];
+  if (!("pageParams" in d))
+    return [];
+  const merged = [];
+  for (const page of pages) {
+    if (Array.isArray(page)) {
+      merged.push(...page);
+    } else if (page && typeof page === "object") {
+      const p = page;
+      const listKeys = ["data", "items", "results", "articles", "posts", "nodes", "records"];
+      let found = false;
+      for (const k of listKeys) {
+        const v = p[k];
+        if (Array.isArray(v)) {
+          merged.push(...v);
+          found = true;
+          break;
+        }
+      }
+      if (!found) {
+        for (const v of Object.values(p)) {
+          if (Array.isArray(v) && v.length > 0) {
+            merged.push(...v);
+            found = true;
+            break;
+          }
+        }
+      }
+      if (!found)
+        merged.push(page);
+    }
+  }
+  return merged;
+}
+function unwrapDehydratedState(pageProps) {
+  if (!pageProps || typeof pageProps !== "object")
+    return [];
+  const dh = pageProps.dehydratedState;
+  if (!dh || typeof dh !== "object")
+    return [];
+  const queries = dh.queries;
+  if (!Array.isArray(queries))
+    return [];
+  const extracted = [];
+  for (const q of queries) {
+    if (!q || typeof q !== "object")
+      continue;
+    const state = q.state;
+    if (!state || typeof state !== "object")
+      continue;
+    const data = state.data;
+    if (data == null)
+      continue;
+    const infinitePages = unwrapInfiniteQuery(data);
+    if (infinitePages.length > 0) {
+      extracted.push(infinitePages);
+    } else {
+      extracted.push(data);
+    }
+  }
+  return extracted;
+}
 function extractSPAData(html) {
   const results = [];
   const nextDataMatch = html.match(/<script\s+id="__NEXT_DATA__"[^>]*>([\s\S]*?)<\/script>/i);
@@ -12416,18 +12632,31 @@ function extractSPAData(html) {
       const parsed = JSON.parse(nextDataMatch[1]);
       const pageProps = parsed?.props?.pageProps;
       if (pageProps && typeof pageProps === "object" && Object.keys(pageProps).length > 0) {
-        results.push({
-          type: "spa-nextjs",
-          data: pageProps,
-          element_count: countDataElements(pageProps)
-        });
+        const dehydrated = unwrapDehydratedState(pageProps);
+        for (const qdata of dehydrated) {
+          if (qdata && typeof qdata === "object") {
+            results.push({
+              type: "spa-nextjs",
+              data: qdata,
+              element_count: countDataElements(qdata)
+            });
+          }
+        }
+        const rawPageProps = dehydrated.length > 0 ? Object.fromEntries(Object.entries(pageProps).filter(([key]) => key !== "dehydratedState")) : pageProps;
+        if (rawPageProps && Object.keys(rawPageProps).length > 0) {
+          results.push({
+            type: "spa-nextjs",
+            data: rawPageProps,
+            element_count: countDataElements(rawPageProps)
+          });
+        }
       }
     } catch {}
   }
-  const nuxtMatch = html.match(/window\.__NUXT__\s*=\s*(\{[\s\S]*?\});?\s*(?:<\/script>|$)/i);
-  if (nuxtMatch) {
+  const nuxtPayload = findWindowAssignmentPayload(html, "__NUXT__");
+  if (nuxtPayload) {
     try {
-      const parsed = JSON.parse(nuxtMatch[1]);
+      const parsed = JSON.parse(nuxtPayload);
       const data = parsed?.data?.[0] ?? parsed?.state ?? parsed;
       if (data && typeof data === "object" && Object.keys(data).length > 0) {
         results.push({
@@ -12438,10 +12667,10 @@ function extractSPAData(html) {
       }
     } catch {}
   }
-  const initialStateMatch = html.match(/window\.__INITIAL_STATE__\s*=\s*(\{[\s\S]*?\});?\s*(?:<\/script>|$)/i);
-  if (initialStateMatch) {
+  const initialStatePayload = findWindowAssignmentPayload(html, "__INITIAL_STATE__");
+  if (initialStatePayload) {
     try {
-      const parsed = JSON.parse(initialStateMatch[1]);
+      const parsed = JSON.parse(initialStatePayload);
       if (parsed && typeof parsed === "object" && Object.keys(parsed).length > 0) {
         results.push({
           type: "spa-initial-state",
@@ -12451,10 +12680,10 @@ function extractSPAData(html) {
       }
     } catch {}
   }
-  const preloadedMatch = html.match(/window\.__PRELOADED_STATE__\s*=\s*(\{[\s\S]*?\});?\s*(?:<\/script>|$)/i);
-  if (preloadedMatch) {
+  const preloadedPayload = findWindowAssignmentPayload(html, "__PRELOADED_STATE__");
+  if (preloadedPayload) {
     try {
-      const parsed = JSON.parse(preloadedMatch[1]);
+      const parsed = JSON.parse(preloadedPayload);
       if (parsed && typeof parsed === "object" && Object.keys(parsed).length > 0) {
         results.push({
           type: "spa-preloaded-state",
@@ -12464,6 +12693,59 @@ function extractSPAData(html) {
       }
     } catch {}
   }
+  const apolloPayload = findWindowAssignmentPayload(html, "__APOLLO_STATE__");
+  if (apolloPayload) {
+    try {
+      const parsed = JSON.parse(apolloPayload);
+      if (parsed && typeof parsed === "object" && Object.keys(parsed).length > 0) {
+        results.push({
+          type: "spa-initial-state",
+          data: parsed,
+          element_count: countDataElements(parsed)
+        });
+      }
+    } catch {}
+  }
+  const nextFPayloads = [];
+  const nextFRe = /self\.__next_f\.push\(\s*\[\s*\d+\s*,\s*("(?:[^"\\]|\\.)*")/g;
+  let nextFMatch;
+  while (nextFMatch = nextFRe.exec(html)) {
+    try {
+      const decoded = JSON.parse(nextFMatch[1]);
+      if (typeof decoded === "string" && decoded.length > 0) {
+        nextFPayloads.push(decoded);
+      }
+    } catch {}
+  }
+  if (nextFPayloads.length > 0) {
+    const combined = nextFPayloads.join("");
+    const fragments = [];
+    for (let i = 0;i < combined.length; i++) {
+      const c = combined[i];
+      if (c !== "{" && c !== "[")
+        continue;
+      const body = sliceBalancedAny(combined, i);
+      if (!body)
+        continue;
+      try {
+        const parsed = JSON.parse(body);
+        if (parsed && typeof parsed === "object") {
+          fragments.push(parsed);
+        }
+      } catch {}
+      i += body.length - 1;
+    }
+    const scored = fragments.filter((f) => f && typeof f === "object").map((f) => ({ data: f, count: countDataElements(f) })).sort((a, b) => b.count - a.count).slice(0, 3);
+    for (const entry of scored) {
+      if (entry.count >= 2) {
+        results.push({
+          type: "spa-initial-state",
+          data: entry.data,
+          element_count: entry.count
+        });
+      }
+    }
+  }
   return results;
 }
 function countDataElements(obj, depth = 0) {
@@ -13421,6 +13703,7 @@ function extractFromDOMWithHint(html, intent, hint) {
   return extractFromDOM(html, intent);
 }
 function extractFromDOM(html, intent) {
+  const spaStructures = extractSPAData(html);
   const MAX_HTML_SIZE = 300000;
   let workingHtml = html;
   if (workingHtml.length > MAX_HTML_SIZE) {
@@ -13434,7 +13717,6 @@ function extractFromDOM(html, intent) {
       }
     }
   }
-  const spaStructures = extractSPAData(workingHtml);
   const flashStructures = extractFlashNoticeSpecial(workingHtml, intent);
   const cleaned = cleanDOM(workingHtml);
   const githubStructures = extractGitHubSpecial(workingHtml, intent);
@@ -15515,6 +15797,7 @@ __export(exports_execution, {
   isBundleInferredEndpoint: () => isBundleInferredEndpoint,
   executeSkill: () => executeSkill,
   executeEndpoint: () => executeEndpoint,
+  detectBrowserBlockSignals: () => detectBrowserBlockSignals,
   deriveStructuredDataReplayUrl: () => deriveStructuredDataReplayUrl,
   deriveStructuredDataReplayTemplate: () => deriveStructuredDataReplayTemplate,
   deriveStructuredDataReplayCandidatesFromInputs: () => deriveStructuredDataReplayCandidatesFromInputs,
@@ -16072,8 +16355,10 @@ function buildPageArtifactCapture(url, intent, html, authRequired = false) {
   }
   const searchForms = detectSearchForms(html);
   const validSearchForm = searchForms.find((spec) => isStructuredSearchForm(spec));
+  const isSpaSource = extracted.extraction_method.startsWith("spa-");
   const response_schema = inferSchema([extracted.data]);
   const computedTemplate = templatizeQueryParams(url);
+  const description = validSearchForm ? `Captured search form artifact for ${intent}` : isSpaSource ? `SSR embedded data (${extracted.extraction_method}) for ${intent}` : `Captured page artifact for ${intent}`;
   const endpoint = {
     endpoint_id: stableEndpointId2("GET", computedTemplate),
     method: "GET",
@@ -16081,7 +16366,7 @@ function buildPageArtifactCapture(url, intent, html, authRequired = false) {
     idempotency: "safe",
     verification_status: "verified",
     reliability_score: extracted.confidence,
-    description: validSearchForm ? `Captured search form artifact for ${intent}` : `Captured page artifact for ${intent}`,
+    description,
     response_schema,
     dom_extraction: {
       extraction_method: extracted.extraction_method,
@@ -16233,7 +16518,8 @@ async function trySeedStructuredDocumentSkill(skill, url, intent, params, target
   };
   let learned = localDraft;
   const validation = await validateManifest({ ...localDraft, skill_id: "__validate__" });
-  if (validation.valid) {
+  const admission = selectMarketplacePublishEndpoints(localDraft);
+  if (validation.valid && admission.endpoints.length > 0) {
     try {
       const { operation_graph: _graph, ...publishDraft } = localDraft;
       const published = await publishSkill3(publishDraft);
@@ -16245,6 +16531,8 @@ async function trySeedStructuredDocumentSkill(skill, url, intent, params, target
     } catch {
       learned = localDraft;
     }
+  } else if (admission.endpoints.length === 0) {
+    console.warn(`[publish] direct publish skipped for ${localDraft.skill_id}: ${admission.stats.by_reason.dom_fallback_only > 0 ? "dom_fallback_only" : "no admitted endpoints"}`);
   }
   try {
     cachePublishedSkill(learned);
@@ -16392,7 +16680,8 @@ async function trySeedPublicDocumentFetchSkill(skill, url, intent, targetDomain,
   };
   let learned = localDraft;
   const validation = await validateManifest({ ...localDraft, skill_id: "__validate__" });
-  if (validation.valid) {
+  const admission = selectMarketplacePublishEndpoints(localDraft);
+  if (validation.valid && admission.endpoints.length > 0) {
     try {
       const { operation_graph: _graph, ...publishDraft } = localDraft;
       const published = await publishSkill3(publishDraft);
@@ -16404,6 +16693,8 @@ async function trySeedPublicDocumentFetchSkill(skill, url, intent, targetDomain,
     } catch {
       learned = localDraft;
     }
+  } else if (admission.endpoints.length === 0) {
+    console.warn(`[publish] direct publish skipped for ${localDraft.skill_id}: ${admission.stats.by_reason.dom_fallback_only > 0 ? "dom_fallback_only" : "no admitted endpoints"}`);
   }
   try {
     cachePublishedSkill(learned);
@@ -16627,7 +16918,62 @@ async function executeBrowserCapture(skill, params, options) {
       };
     }
   }
-  const endpoints = extractEndpoints(captured.requests, captured.ws_messages, { pageUrl: url, finalUrl: captured.final_url, intent });
+  const extractionTrace = {};
+  const endpoints = extractEndpoints(captured.requests, captured.ws_messages, { pageUrl: url, finalUrl: captured.final_url, intent }, extractionTrace);
+  const computeCapturedMeta = () => {
+    const html = captured.html ?? "";
+    const titleMatch = html.toLowerCase().match(/<title[^>]*>([^<]{0,200})<\/title>/);
+    const title = titleMatch ? titleMatch[1].trim() : "";
+    const stripped = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");
+    const text = stripped.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
+    let intentVerdict = "skip";
+    let intentReason = "no_semantic_assessment";
+    if (text && intent) {
+      try {
+        const assessment = assessIntentResult(text, intent);
+        intentVerdict = assessment.verdict;
+        intentReason = assessment.reason;
+      } catch {}
+    }
+    const rows = extractionTrace.rows ?? [];
+    const rejectionCounts = {};
+    const samplesByReason = {};
+    const PER_REASON_SAMPLE_CAP = 5;
+    for (const row of rows) {
+      if (row.kept === true)
+        continue;
+      const reason = String(row.reason ?? "unknown");
+      rejectionCounts[reason] = (rejectionCounts[reason] ?? 0) + 1;
+      if (typeof row.url === "string") {
+        const bucket = samplesByReason[reason] ?? (samplesByReason[reason] = []);
+        if (bucket.length < PER_REASON_SAMPLE_CAP)
+          bucket.push(row.url);
+      }
+    }
+    const rejectedSamples = [];
+    for (const [reason, urls] of Object.entries(samplesByReason)) {
+      for (const u of urls)
+        rejectedSamples.push({ url: u, reason });
+    }
+    const apiCallCount = captured.requests?.length ?? 0;
+    const blockSignals = detectBrowserBlockSignals({
+      requestUrls: (captured.requests ?? []).map((r) => r.url ?? ""),
+      title,
+      htmlLength: html.length,
+      rejectionCounts
+    });
+    return {
+      html_bytes: html.length,
+      title,
+      text_bytes: text.length,
+      observed_api_calls: apiCallCount,
+      intent_verdict: intentVerdict,
+      intent_reason: intentReason,
+      filter_rejections: rejectionCounts,
+      rejected_samples: rejectedSamples,
+      browser_block_signals: blockSignals
+    };
+  };
   if (captured.html) {
     const detectedForms = detectSearchForms(captured.html);
     if (detectedForms.length > 0) {
@@ -16796,8 +17142,11 @@ async function executeBrowserCapture(skill, params, options) {
       let learned2 = domDraft;
       try {
         const validation = await validateManifest({ ...domDraft, skill_id: "__validate__" });
-        if (validation.valid) {
+        const admission = selectMarketplacePublishEndpoints(domDraft);
+        if (validation.valid && admission.endpoints.length > 0) {
           learned2 = await publishSkill3(domDraft);
+        } else if (admission.endpoints.length === 0) {
+          console.warn(`[publish] dom-artifact publish skipped for ${domDraft.skill_id}: dom_fallback_only (kept local-only)`);
         }
       } catch {}
       if (learned2) {
@@ -16835,34 +17184,12 @@ async function executeBrowserCapture(skill, params, options) {
         trace: trace3,
         result: {
           error: "low_quality_dom_extraction",
-          message: `Structured DOM extraction was rejected for ${url}: ${pageArtifact.quality_note}`
+          message: `Structured DOM extraction was rejected for ${url}: ${pageArtifact.quality_note}`,
+          captured_meta: computeCapturedMeta()
         }
       };
     }
-    const capturedMeta = (() => {
-      const html = captured.html ?? "";
-      const titleMatch = html.toLowerCase().match(/<title[^>]*>([^<]{0,200})<\/title>/);
-      const title = titleMatch ? titleMatch[1].trim() : "";
-      const stripped = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "").replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");
-      const text = stripped.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").trim();
-      let intentVerdict = "skip";
-      let intentReason = "no_semantic_assessment";
-      if (text && intent) {
-        try {
-          const assessment = assessIntentResult(text, intent);
-          intentVerdict = assessment.verdict;
-          intentReason = assessment.reason;
-        } catch {}
-      }
-      return {
-        html_bytes: html.length,
-        title,
-        text_bytes: text.length,
-        observed_api_calls: captured.requests?.length ?? 0,
-        intent_verdict: intentVerdict,
-        intent_reason: intentReason
-      };
-    })();
+    const capturedMeta = computeCapturedMeta();
     const trace2 = stampTrace({
       trace_id: traceId,
       skill_id: skill.skill_id,
@@ -18077,6 +18404,51 @@ function semanticIntentAdjustment(endpoint, intent) {
   }
   return delta;
 }
+function detectBrowserBlockSignals(input) {
+  const { requestUrls, title, htmlLength, rejectionCounts } = input;
+  const signals = [];
+  const titleLower = title.toLowerCase();
+  if (/just a moment|attention required|access denied|pardon our interruption|captcha|verifying you are human|human verification|are you a robot|bot check|cloudflare|press and hold|request could not be satisfied|403 forbidden|\b404\b|\b502\b|\b503\b|\b504\b|bad gateway|service unavailable|gateway timeout|site blocked|unusual traffic|security check|not[ _.]?found|page (does )?not exist|page doesn't exist|this page can't be|server error/i.test(titleLower)) {
+    signals.push("challenge_title");
+  }
+  const vendorHits = new Set;
+  for (const u of requestUrls) {
+    if (/perimeterx|px-cloud|px-cdn|pxhd\.net/i.test(u) || /KP_UIDz=/.test(u) || /\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\/(ips\.js|tl|xhr|init)/i.test(u)) {
+      vendorHits.add("perimeterx");
+    }
+    if (/datadome|js\.datadome|dd\.datadome|_dd\.s|ddjskey/i.test(u))
+      vendorHits.add("datadome");
+    if (/akamaihd|ak-challenge|_Incapsula|incapsula|reese84/i.test(u))
+      vendorHits.add("imperva_incapsula");
+    if (/akam\.net|bot-defender|\/_bm\/|sensor[-_]data|bm\.nuid|_abck/i.test(u))
+      vendorHits.add("akamai_bot_manager");
+    if (/cf-challenge|__cf_chl_|turnstile|challenges\.cloudflare/i.test(u))
+      vendorHits.add("cloudflare");
+    if (/hcaptcha|recaptcha|arkoselabs|funcaptcha/i.test(u))
+      vendorHits.add("captcha_vendor");
+    if (/shape\.security|f5\.com\/shape|ShapeSecurity/i.test(u))
+      vendorHits.add("shape_security");
+    if (/kasada|client\.kasada|ips\.kasada/i.test(u))
+      vendorHits.add("kasada");
+  }
+  for (const v of vendorHits)
+    signals.push(`vendor:${v}`);
+  const apiCallCount = requestUrls.length;
+  const noisyRejections = (rejectionCounts.not_api_like ?? 0) + (rejectionCounts.score_non_positive ?? 0);
+  if (apiCallCount > 0 && apiCallCount <= 20 && noisyRejections >= Math.max(1, Math.floor(apiCallCount * 0.6))) {
+    signals.push("sparse_capture_mostly_noise");
+  }
+  if (htmlLength < 500 && apiCallCount === 0) {
+    signals.push("empty_capture");
+  }
+  if (htmlLength < 500 && apiCallCount >= 30) {
+    signals.push("no_html_many_apis");
+  }
+  if (htmlLength < 500 && apiCallCount > 0 && apiCallCount < 30) {
+    signals.push("low_capture");
+  }
+  return signals;
+}
 function rankEndpoints(endpoints, intent, skillDomain, contextUrl) {
   const NOISE_HOSTS = /(id5-sync\.com|btloader\.com|presage\.io|onetrust\.com|adsrvr\.org|googlesyndication\.com|adtrafficquality\.google|amazon-adsystem\.com|crazyegg\.com|challenges\.cloudflare\.com|google-analytics\.com|doubleclick\.net|gstatic\.com|accounts\.google\.com|login\.microsoftonline\.com|auth0\.com|cognito-idp\.|protechts\.net|demdex\.net|datadoghq\.com|fullstory\.com|launchdarkly\.com|intercom\.io|sentry\.io|segment\.io|amplitude\.com|mixpanel\.com|hotjar\.com|clarity\.ms|googletagmanager\.com|walletconnect\.com|cloudflareinsights\.com|fonts\.googleapis\.com|recaptcha|waa-pa\.|signaler-pa\.|ogads-pa\.|reddit\.com\/pixels?|pixel-config\.|dns-finder\.com|cookieconsentpub|firebase\.googleapis\.com|firebaseinstallations\.googleapis\.com|identitytoolkit\.googleapis\.com|securetoken\.googleapis\.com|apis\.google\.com|connect\.facebook\.net|bat\.bing\.com|static\.cloudflareinsights\.com|cdn\.mxpnl\.com|js\.hs-analytics\.net|snap\.licdn\.com|clc\.stackoverflow\.com|px\.ads|t\.co\/i|analytics\.|telemetry\.|stats\.)/i;
   const NOISE_PATHS = /\/(track|pixel|telemetry|beacon|csp-report|litms|demdex|analytics|protechts|collect|tr\/|gen_204|generate_204|log$|logging|heartbeat|metrics|consent|sodar|tag$|event$|events$|impression|pageview|click|__|adx\/|\/cm\/ttc|\/pfb$|_stm$|videoads\/|prerolls|phantom\/)/i;
@@ -18209,7 +18581,7 @@ function rankEndpoints(endpoints, intent, skillDomain, contextUrl) {
       score += bm25Score(queryTokens, docs[i], avgDl, docCount, docFreqs) * 20;
     }
     if (descriptionMeta.source === "agent" && descriptionMeta.display && rawTokens.length > 0) {
-      const descTokens = new Set(descriptionMeta.display.toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((w) => w.length > 1 && !STOPWORDS.has(w)).map((w) => stem(w)));
+      const descTokens = new Set(descriptionMeta.display.replace(/([a-z0-9])([A-Z])/g, "$1 $2").replace(/([A-Z]+)([A-Z][a-z])/g, "$1 $2").toLowerCase().replace(/[^a-z0-9]+/g, " ").split(/\s+/).filter((w) => w.length > 1 && !STOPWORDS.has(w)).map((w) => stem(w)));
       const rawStems = new Set(rawTokens.map((t) => stem(t)));
       let matches = 0;
       for (const t of rawStems) {
@@ -18479,6 +18851,7 @@ var init_execution = __esm(async () => {
   init_bundle_scanner();
   init_token_resolver();
   init_marketplace();
+  init_publish_admission();
   init_transform();
   init_drift();
   init_client2();
@@ -20131,7 +20504,19 @@ function isCachedSkillRelevantForIntent(skill, intent, contextUrl) {
     if (collectExplicitSearchContextBindingKeys(contextUrl).size > 0)
       return false;
   }
-  return (top?.score ?? Number.NEGATIVE_INFINITY) >= 0;
+  if ((top?.score ?? Number.NEGATIVE_INFINITY) >= 0)
+    return true;
+  if (top && top.score >= -5 && contextUrl) {
+    try {
+      const epHost = new URL(top.endpoint.url_template).hostname;
+      const ctxHost = new URL(contextUrl).hostname;
+      const epReg = getRegistrableDomain(epHost);
+      const ctxReg = getRegistrableDomain(ctxHost);
+      if (epReg && ctxReg && epReg === ctxReg)
+        return true;
+    } catch {}
+  }
+  return false;
 }
 function assessLocalExecutionResult(endpoint, result, intent, trace) {
   const semanticAssessment = assessIntentResult(result, intent);
@@ -22617,6 +23002,8 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
       error: `No relevant endpoint discovered for "${queryIntent}"`
     };
     console.warn(`[capture] dropping learned skill with no relevant endpoints for "${queryIntent}"`);
+    const totalEndpoints = resolvedSkill.endpoints.length;
+    const captureDiagnostic = totalEndpoints === 0 ? "no_endpoints_extracted" : ranked.length === 0 ? "all_endpoints_filtered_by_noise_rules" : "endpoints_scored_below_relevance_threshold";
     return {
       result: {
         error: `No relevant endpoint discovered for "${queryIntent}"`,
@@ -22626,6 +23013,8 @@ async function resolveAndExecute(intent, params = {}, context, projection, optio
           description: candidate.endpoint.description,
           url: candidate.endpoint.url_template
         })),
+        capture_diagnostic: captureDiagnostic,
+        total_endpoints_captured: totalEndpoints,
         ...authRecommended ? {
           auth_recommended: true,
           auth_hint: captureResult?.auth_hint

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "unbrowse",
-  "version": "3.8.0-preview.2",
+  "version": "3.8.0-preview.3",
   "description": "Reverse-engineer any website into reusable API skills. Zero-dep single binary with embedded browser engine.",
   "type": "module",
   "bin": {